diff --git a/README.md b/README.md index c764730..6f997ed 100644 --- a/README.md +++ b/README.md @@ -117,18 +117,19 @@ scotty beam listens deezer listenbrainz --timestamp "2023-12-06 14:26:24" ### Supported backends The following table lists the available backends and the currently supported features. -Backend | Listens Export | Listens Import | Loves Export | Loves Import -----------------|----------------|----------------|--------------|------------- -deezer | ✓ | ⨯ | ✓ | - -funkwhale | ✓ | ⨯ | ✓ | - -jspf | ✓ | ✓ | ✓ | ✓ -lastfm | ✓ | ✓ | ✓ | ✓ -listenbrainz | ✓ | ✓ | ✓ | ✓ -maloja | ✓ | ✓ | ⨯ | ⨯ -scrobbler-log | ✓ | ✓ | ⨯ | ⨯ -spotify | ✓ | ⨯ | ✓ | - -spotify-history | ✓ | ⨯ | ⨯ | ⨯ -subsonic | ⨯ | ⨯ | ✓ | - +Backend | Listens Export | Listens Import | Loves Export | Loves Import +---------------------|----------------|----------------|--------------|------------- +deezer | ✓ | ⨯ | ✓ | - +funkwhale | ✓ | ⨯ | ✓ | - +jspf | ✓ | ✓ | ✓ | ✓ +lastfm | ✓ | ✓ | ✓ | ✓ +listenbrainz | ✓ | ✓ | ✓ | ✓ +listenbrainz-archive | ✓ | - | - | - +maloja | ✓ | ✓ | ⨯ | ⨯ +scrobbler-log | ✓ | ✓ | ⨯ | ⨯ +spotify | ✓ | ⨯ | ✓ | - +spotify-history | ✓ | ⨯ | ⨯ | ⨯ +subsonic | ⨯ | ⨯ | ✓ | - **✓** implemented **-** not yet implemented **⨯** unavailable / not planned diff --git a/config.example.toml b/config.example.toml index 6b81bac..40ffd18 100644 --- a/config.example.toml +++ b/config.example.toml @@ -19,6 +19,13 @@ token = "" # not already exists in your ListenBrainz profile. check-duplicate-listens = false +[service.listenbrainz-archive] +# This backend supports listens from a ListenBrainz export archive +# (https://listenbrainz.org/settings/export/). +backend = "listenbrainz-archive" +# The file path to the ListenBrainz export archive. +file-path = "./listenbrainz_outsidecontext.zip" + [service.maloja] # Maloja is a self hosted listening service (https://github.com/krateng/maloja) backend = "maloja" diff --git a/go.mod b/go.mod index a00b416..ccdb6cc 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.9.0 // indirect + github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.6 // indirect diff --git a/go.sum b/go.sum index 3cd01a6..028515c 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,8 @@ github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFT github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0 h1:cgqwZtnR+IQfUYDLJ3Kiy4aE+O/wExTzEIg8xwC4Qfs= github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0/go.mod h1:n3nudMl178cEvD44PaopxH9jhJaQzthSxUzLO5iKMy4= +github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 h1:CXJI+lliMiiEwzfgE8yt/38K0heYDgQ0L3f/3fxRnQU= +github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740/go.mod h1:G4w16caPmc6at7u4fmkj/8OAoOnM9mkmJr2fvL0vhaw= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 0000000..7714552 --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,179 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +// Implements generic access to files inside an archive. +// +// An archive in this context can be any container that holds files. +// In this implementation the archive can be a ZIP file or a directory. +package archive + +import ( + "archive/zip" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" +) + +// Generic archive interface. +type Archive interface { + Close() error + Open(path string) (fs.File, error) + Glob(pattern string) ([]FileInfo, error) +} + +// Open an archive in path. +// The archive can be a ZIP file or a directory. The implementation +// will detect the type of archive and return the appropriate +// implementation of the Archive interface. +func OpenArchive(path string) (Archive, error) { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + switch mode := fi.Mode(); { + case mode.IsRegular(): + archive := &zipArchive{} + err := archive.OpenArchive(path) + if err != nil { + return nil, err + } + return archive, nil + case mode.IsDir(): + archive := &dirArchive{} + err := archive.OpenArchive(path) + if err != nil { + return nil, err + } + return archive, nil + default: + return nil, fmt.Errorf("unsupported file mode: %s", mode) + } +} + +// Interface for a file that can be opened when needed. +type OpenableFile interface { + Open() (io.ReadCloser, error) +} + +// Generic information about a file inside an archive. +type FileInfo struct { + Name string + File OpenableFile +} + +// A openable file in the filesystem. +type filesystemFile struct { + path string +} + +func (f *filesystemFile) Open() (io.ReadCloser, error) { + return os.Open(f.path) +} + +// An implementation of the archiveBackend interface for zip files. +type zipArchive struct { + zip *zip.ReadCloser +} + +func (a *zipArchive) OpenArchive(path string) error { + zip, err := zip.OpenReader(path) + if err != nil { + return err + } + a.zip = zip + return nil +} + +func (a *zipArchive) Close() error { + if a.zip == nil { + return nil + } + return a.zip.Close() +} + +func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { + result := make([]FileInfo, 0) + for _, file := range a.zip.File { + if matched, err := filepath.Match(pattern, file.Name); matched { + if err != nil { + return nil, err + } + info := FileInfo{ + Name: file.Name, + File: file, + } + result = append(result, info) + } + } + + return result, nil +} + +func (a *zipArchive) Open(path string) (fs.File, error) { + file, err := a.zip.Open(path) + if err != nil { + return nil, err + } + return file, nil +} + +// An implementation of the archiveBackend interface for directories. +type dirArchive struct { + path string + dirFS fs.FS +} + +func (a *dirArchive) OpenArchive(path string) error { + a.path = filepath.Clean(path) + a.dirFS = os.DirFS(path) + return nil +} + +func (a *dirArchive) Close() error { + return nil +} + +// Open opens the named file in the archive. +// [fs.File.Close] must be called to release any associated resources. +func (a *dirArchive) Open(path string) (fs.File, error) { + return a.dirFS.Open(path) +} + +func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { + files, err := fs.Glob(a.dirFS, pattern) + if err != nil { + return nil, err + } + result := make([]FileInfo, 0) + for _, name := range files { + fullPath := filepath.Join(a.path, name) + info := FileInfo{ + Name: name, + File: &filesystemFile{path: fullPath}, + } + result = append(result, info) + } + + return result, nil +} diff --git a/internal/backends/backends.go b/internal/backends/backends.go index a9c3292..a1cd407 100644 --- a/internal/backends/backends.go +++ b/internal/backends/backends.go @@ -27,6 +27,7 @@ import ( "go.uploadedlobster.com/scotty/internal/backends/funkwhale" "go.uploadedlobster.com/scotty/internal/backends/jspf" "go.uploadedlobster.com/scotty/internal/backends/lastfm" + "go.uploadedlobster.com/scotty/internal/backends/lbarchive" "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/backends/maloja" "go.uploadedlobster.com/scotty/internal/backends/scrobblerlog" @@ -105,17 +106,18 @@ func GetBackends() BackendList { } var knownBackends = map[string]func() models.Backend{ - "deezer": func() models.Backend { return &deezer.DeezerApiBackend{} }, - "dump": func() models.Backend { return &dump.DumpBackend{} }, - "funkwhale": func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} }, - "jspf": func() models.Backend { return &jspf.JSPFBackend{} }, - "lastfm": func() models.Backend { return &lastfm.LastfmApiBackend{} }, - "listenbrainz": func() models.Backend { return &listenbrainz.ListenBrainzApiBackend{} }, - "maloja": func() models.Backend { return &maloja.MalojaApiBackend{} }, - "scrobbler-log": func() models.Backend { return &scrobblerlog.ScrobblerLogBackend{} }, - "spotify": func() models.Backend { return &spotify.SpotifyApiBackend{} }, - "spotify-history": func() models.Backend { return &spotifyhistory.SpotifyHistoryBackend{} }, - "subsonic": func() models.Backend { return &subsonic.SubsonicApiBackend{} }, + "deezer": func() models.Backend { return &deezer.DeezerApiBackend{} }, + "dump": func() models.Backend { return &dump.DumpBackend{} }, + "funkwhale": func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} }, + "jspf": func() models.Backend { return &jspf.JSPFBackend{} }, + "lastfm": func() models.Backend { return &lastfm.LastfmApiBackend{} }, + "listenbrainz": func() models.Backend { return &listenbrainz.ListenBrainzApiBackend{} }, + "listenbrainz-archive": func() models.Backend { return &lbarchive.ListenBrainzArchiveBackend{} }, + "maloja": func() models.Backend { return &maloja.MalojaApiBackend{} }, + "scrobbler-log": func() models.Backend { return &scrobblerlog.ScrobblerLogBackend{} }, + "spotify": func() models.Backend { return &spotify.SpotifyApiBackend{} }, + "spotify-history": func() models.Backend { return &spotifyhistory.SpotifyHistoryBackend{} }, + "subsonic": func() models.Backend { return &subsonic.SubsonicApiBackend{} }, } func backendWithConfig(config config.ServiceConfig) (models.Backend, error) { diff --git a/internal/backends/backends_test.go b/internal/backends/backends_test.go index e115636..737c7e3 100644 --- a/internal/backends/backends_test.go +++ b/internal/backends/backends_test.go @@ -28,6 +28,7 @@ import ( "go.uploadedlobster.com/scotty/internal/backends/funkwhale" "go.uploadedlobster.com/scotty/internal/backends/jspf" "go.uploadedlobster.com/scotty/internal/backends/lastfm" + "go.uploadedlobster.com/scotty/internal/backends/lbarchive" "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/backends/maloja" "go.uploadedlobster.com/scotty/internal/backends/scrobblerlog" @@ -103,6 +104,11 @@ func TestImplementsInterfaces(t *testing.T) { expectInterface[models.LovesExport](t, &lastfm.LastfmApiBackend{}) expectInterface[models.LovesImport](t, &lastfm.LastfmApiBackend{}) + expectInterface[models.ListensExport](t, &lbarchive.ListenBrainzArchiveBackend{}) + // expectInterface[models.ListensImport](t, &lbarchive.ListenBrainzArchiveBackend{}) + // expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{}) + // expectInterface[models.LovesImport](t, &lbarchive.ListenBrainzArchiveBackend{}) + expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{}) expectInterface[models.ListensImport](t, &listenbrainz.ListenBrainzApiBackend{}) expectInterface[models.LovesExport](t, &listenbrainz.ListenBrainzApiBackend{}) diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go new file mode 100644 index 0000000..0848d38 --- /dev/null +++ b/internal/backends/lbarchive/lbarchive.go @@ -0,0 +1,121 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package lbarchive + +import ( + "context" + "time" + + lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" + "go.uploadedlobster.com/scotty/internal/config" + "go.uploadedlobster.com/scotty/internal/i18n" + "go.uploadedlobster.com/scotty/internal/listenbrainz" + "go.uploadedlobster.com/scotty/internal/models" +) + +const batchSize = 2000 + +type ListenBrainzArchiveBackend struct { + filePath string +} + +func (b *ListenBrainzArchiveBackend) Name() string { return "listenbrainz-archive" } + +func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption { + return []models.BackendOption{{ + Name: "file-path", + Label: i18n.Tr("Export ZIP file path"), + Type: models.String, + }} +} + +func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error { + b.filePath = config.GetString("file-path") + return nil +} + +func (b *ListenBrainzArchiveBackend) ExportListens( + ctx context.Context, oldestTimestamp time.Time, + results chan models.ListensResult, progress chan models.TransferProgress) { + startTime := time.Now() + minTime := oldestTimestamp + if minTime.Unix() < 1 { + minTime = time.Unix(1, 0) + } + + totalDuration := startTime.Sub(oldestTimestamp) + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(totalDuration.Seconds()), + }, + } + + archive, err := listenbrainz.OpenExportArchive(b.filePath) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + defer archive.Close() + + userInfo, err := archive.UserInfo() + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + + listens := make(models.ListensList, 0, batchSize) + for rawListen, err := range archive.IterListens(oldestTimestamp) { + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + + listen := lbapi.AsListen(rawListen) + if listen.UserName == "" { + listen.UserName = userInfo.Name + } + listens = append(listens, listen) + + // Update the progress + p.Export.TotalItems += 1 + remainingTime := startTime.Sub(listen.ListenedAt) + p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) + + // Allow the importer to start processing the listens by + // sending them in batches. + if len(listens) >= batchSize { + results <- models.ListensResult{Items: listens} + progress <- p + listens = listens[:0] + } + } + + results <- models.ListensResult{Items: listens} + p.Export.Complete() + progress <- p +} diff --git a/internal/backends/lbarchive/lbarchive_test.go b/internal/backends/lbarchive/lbarchive_test.go new file mode 100644 index 0000000..b7e164a --- /dev/null +++ b/internal/backends/lbarchive/lbarchive_test.go @@ -0,0 +1,40 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package lbarchive_test + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "go.uploadedlobster.com/scotty/internal/backends/lbarchive" + "go.uploadedlobster.com/scotty/internal/config" +) + +func TestInitConfig(t *testing.T) { + c := viper.New() + c.Set("file-path", "/foo/lbarchive.zip") + service := config.NewServiceConfig("test", c) + backend := lbarchive.ListenBrainzArchiveBackend{} + err := backend.InitConfig(&service) + assert.NoError(t, err) +} diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index bf46c22..4f0ce2f 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -26,13 +26,14 @@ import ( "go.uploadedlobster.com/musicbrainzws2" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" + "go.uploadedlobster.com/scotty/internal/listenbrainz" "go.uploadedlobster.com/scotty/internal/models" "go.uploadedlobster.com/scotty/internal/similarity" "go.uploadedlobster.com/scotty/internal/version" ) type ListenBrainzApiBackend struct { - client Client + client listenbrainz.Client mbClient musicbrainzws2.Client username string checkDuplicates bool @@ -58,13 +59,13 @@ func (b *ListenBrainzApiBackend) Options() []models.BackendOption { } func (b *ListenBrainzApiBackend) InitConfig(config *config.ServiceConfig) error { - b.client = NewClient(config.GetString("token")) + b.client = listenbrainz.NewClient(config.GetString("token"), version.UserAgent()) b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ Name: version.AppName, Version: version.AppVersion, URL: version.AppURL, }) - b.client.MaxResults = MaxItemsPerGet + b.client.MaxResults = listenbrainz.MaxItemsPerGet b.username = config.GetString("username") b.checkDuplicates = config.GetBool("check-duplicate-listens", false) return nil @@ -116,7 +117,7 @@ func (b *ListenBrainzApiBackend) ExportListens(ctx context.Context, oldestTimest for _, listen := range result.Payload.Listens { if listen.ListenedAt > oldestTimestamp.Unix() { - listens = append(listens, listen.AsListen()) + listens = append(listens, AsListen(listen)) } else { // result contains listens older then oldestTimestamp break @@ -138,16 +139,16 @@ func (b *ListenBrainzApiBackend) ExportListens(ctx context.Context, oldestTimest func (b *ListenBrainzApiBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { total := len(export.Items) p := models.TransferProgress{}.FromImportResult(importResult, false) - for i := 0; i < total; i += MaxListensPerRequest { - listens := export.Items[i:min(i+MaxListensPerRequest, total)] + for i := 0; i < total; i += listenbrainz.MaxListensPerRequest { + listens := export.Items[i:min(i+listenbrainz.MaxListensPerRequest, total)] count := len(listens) if count == 0 { break } - submission := ListenSubmission{ - ListenType: Import, - Payload: make([]Listen, 0, count), + submission := listenbrainz.ListenSubmission{ + ListenType: listenbrainz.Import, + Payload: make([]listenbrainz.Listen, 0, count), } for _, l := range listens { @@ -167,9 +168,9 @@ func (b *ListenBrainzApiBackend) ImportListens(ctx context.Context, export model } l.FillAdditionalInfo() - listen := Listen{ + listen := listenbrainz.Listen{ ListenedAt: l.ListenedAt.Unix(), - TrackMetadata: Track{ + TrackMetadata: listenbrainz.Track{ TrackName: l.TrackName, ReleaseName: l.ReleaseName, ArtistName: l.ArtistName(), @@ -228,7 +229,7 @@ func (b *ListenBrainzApiBackend) ExportLoves(ctx context.Context, oldestTimestam func (b *ListenBrainzApiBackend) exportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult) { offset := 0 defer close(results) - loves := make(models.LovesList, 0, 2*MaxItemsPerGet) + loves := make(models.LovesList, 0, 2*listenbrainz.MaxItemsPerGet) out: for { @@ -254,7 +255,7 @@ out: } } - love := feedback.AsLove() + love := AsLove(feedback) if love.Created.After(oldestTimestamp) { loves = append(loves, love) } else { @@ -262,7 +263,7 @@ out: } } - offset += MaxItemsPerGet + offset += listenbrainz.MaxItemsPerGet } sort.Sort(loves) @@ -278,7 +279,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(ctx context.Context, export models. go b.exportLoves(ctx, time.Unix(0, 0), existingLovesChan) // TODO: Store MBIDs directly - b.existingMBIDs = make(map[mbtypes.MBID]bool, MaxItemsPerGet) + b.existingMBIDs = make(map[mbtypes.MBID]bool, listenbrainz.MaxItemsPerGet) for existingLoves := range existingLovesChan { if existingLoves.Error != nil { @@ -316,7 +317,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(ctx context.Context, export models. if b.existingMBIDs[recordingMBID] { ok = true } else { - resp, err := b.client.SendFeedback(ctx, Feedback{ + resp, err := b.client.SendFeedback(ctx, listenbrainz.Feedback{ RecordingMBID: recordingMBID, Score: 1, }) @@ -366,7 +367,7 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste } for _, c := range candidates.Payload.Listens { - sim := similarity.CompareTracks(listen.Track, c.TrackMetadata.AsTrack()) + sim := similarity.CompareTracks(listen.Track, AsTrack(c.TrackMetadata)) if sim >= trackSimilarityThreshold { return true, nil } @@ -375,7 +376,8 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste return false, nil } -func (b *ListenBrainzApiBackend) lookupRecording(ctx context.Context, mbid mbtypes.MBID) (*Track, error) { +func (b *ListenBrainzApiBackend) lookupRecording( + ctx context.Context, mbid mbtypes.MBID) (*listenbrainz.Track, error) { filter := musicbrainzws2.IncludesFilter{ Includes: []string{"artist-credits"}, } @@ -388,10 +390,10 @@ func (b *ListenBrainzApiBackend) lookupRecording(ctx context.Context, mbid mbtyp for _, artist := range recording.ArtistCredit { artistMBIDs = append(artistMBIDs, artist.Artist.ID) } - track := Track{ + track := listenbrainz.Track{ TrackName: recording.Title, ArtistName: recording.ArtistCredit.String(), - MBIDMapping: &MBIDMapping{ + MBIDMapping: &listenbrainz.MBIDMapping{ // In case of redirects this MBID differs from the looked up MBID RecordingMBID: recording.ID, ArtistMBIDs: artistMBIDs, @@ -400,26 +402,26 @@ func (b *ListenBrainzApiBackend) lookupRecording(ctx context.Context, mbid mbtyp return &track, nil } -func (lbListen Listen) AsListen() models.Listen { +func AsListen(lbListen listenbrainz.Listen) models.Listen { listen := models.Listen{ ListenedAt: time.Unix(lbListen.ListenedAt, 0), UserName: lbListen.UserName, - Track: lbListen.TrackMetadata.AsTrack(), + Track: AsTrack(lbListen.TrackMetadata), } return listen } -func (f Feedback) AsLove() models.Love { +func AsLove(f listenbrainz.Feedback) models.Love { recordingMBID := f.RecordingMBID track := f.TrackMetadata if track == nil { - track = &Track{} + track = &listenbrainz.Track{} } love := models.Love{ UserName: f.UserName, RecordingMBID: recordingMBID, Created: time.Unix(f.Created, 0), - Track: track.AsTrack(), + Track: AsTrack(*track), } if love.Track.RecordingMBID == "" { @@ -429,7 +431,7 @@ func (f Feedback) AsLove() models.Love { return love } -func (t Track) AsTrack() models.Track { +func AsTrack(t listenbrainz.Track) models.Track { track := models.Track{ TrackName: t.TrackName, ReleaseName: t.ReleaseName, diff --git a/internal/backends/listenbrainz/listenbrainz_test.go b/internal/backends/listenbrainz/listenbrainz_test.go index bf2e4d3..f7151e5 100644 --- a/internal/backends/listenbrainz/listenbrainz_test.go +++ b/internal/backends/listenbrainz/listenbrainz_test.go @@ -24,15 +24,16 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" + lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" + "go.uploadedlobster.com/scotty/internal/listenbrainz" ) func TestInitConfig(t *testing.T) { c := viper.New() c.Set("token", "thetoken") service := config.NewServiceConfig("test", c) - backend := listenbrainz.ListenBrainzApiBackend{} + backend := lbapi.ListenBrainzApiBackend{} err := backend.InitConfig(&service) assert.NoError(t, err) } @@ -57,7 +58,7 @@ func TestListenBrainzListenAsListen(t *testing.T) { }, }, } - listen := lbListen.AsListen() + listen := lbapi.AsListen(lbListen) assert.Equal(t, time.Unix(1699289873, 0), listen.ListenedAt) assert.Equal(t, lbListen.UserName, listen.UserName) assert.Equal(t, time.Duration(413787*time.Millisecond), listen.Duration) @@ -93,7 +94,7 @@ func TestListenBrainzFeedbackAsLove(t *testing.T) { }, }, } - love := feedback.AsLove() + love := lbapi.AsLove(feedback) assert := assert.New(t) assert.Equal(time.Unix(1699859066, 0).Unix(), love.Created.Unix()) assert.Equal(feedback.UserName, love.UserName) @@ -114,7 +115,7 @@ func TestListenBrainzPartialFeedbackAsLove(t *testing.T) { RecordingMBID: recordingMBID, Score: 1, } - love := feedback.AsLove() + love := lbapi.AsLove(feedback) assert := assert.New(t) assert.Equal(time.Unix(1699859066, 0).Unix(), love.Created.Unix()) assert.Equal(recordingMBID, love.RecordingMBID) diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index 76d0c9e..ce470ff 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -20,7 +20,6 @@ package spotifyhistory import ( "context" "os" - "path" "path/filepath" "slices" "sort" @@ -74,7 +73,7 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error { } func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { - files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob)) + files, err := filepath.Glob(filepath.Join(b.dirPath, historyFileGlob)) p := models.TransferProgress{ Export: &models.Progress{}, } diff --git a/internal/config/config.go b/internal/config/config.go index a529b92..94da799 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "os" - "path" "path/filepath" "regexp" "strings" @@ -40,7 +39,7 @@ const ( func DefaultConfigDir() string { configDir, err := os.UserConfigDir() cobra.CheckErr(err) - return path.Join(configDir, version.AppName) + return filepath.Join(configDir, version.AppName) } // initConfig reads in config file and ENV variables if set. diff --git a/internal/listenbrainz/archive.go b/internal/listenbrainz/archive.go new file mode 100644 index 0000000..b7b5909 --- /dev/null +++ b/internal/listenbrainz/archive.go @@ -0,0 +1,251 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package listenbrainz + +import ( + "encoding/json" + "errors" + "io" + "iter" + "regexp" + "sort" + "strconv" + "time" + + "github.com/simonfrey/jsonl" + "go.uploadedlobster.com/scotty/internal/archive" +) + +// Represents a ListenBrainz export archive. +// +// The export contains the user's listen history, favorite tracks and +// user information. +type ExportArchive struct { + backend archive.Archive +} + +// Open a ListenBrainz archive from file path. +func OpenExportArchive(path string) (*ExportArchive, error) { + backend, err := archive.OpenArchive(path) + if err != nil { + return nil, err + } + + return &ExportArchive{backend: backend}, nil +} + +// Close the archive and release any resources. +func (a *ExportArchive) Close() error { + if a.backend == nil { + return nil + } + return a.backend.Close() +} + +// Read the user information from the archive. +func (a *ExportArchive) UserInfo() (UserInfo, error) { + f, err := a.backend.Open("user.json") + if err != nil { + return UserInfo{}, err + } + defer f.Close() + + userInfo := UserInfo{} + bytes, err := io.ReadAll(f) + if err != nil { + return userInfo, err + } + + json.Unmarshal(bytes, &userInfo) + return userInfo, nil +} + +func (a *ExportArchive) ListListenExports() ([]ListenExportFileInfo, error) { + re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`) + result := make([]ListenExportFileInfo, 0) + + files, err := a.backend.Glob("listens/*/*.jsonl") + if err != nil { + return nil, err + } + + for _, file := range files { + match := re.FindStringSubmatch(file.Name) + if match == nil { + continue + } + + year := match[1] + month := match[2] + times, err := getMonthTimeRange(year, month) + if err != nil { + return nil, err + } + info := ListenExportFileInfo{ + Name: file.Name, + TimeRange: *times, + f: file.File, + } + result = append(result, info) + } + + return result, nil +} + +// Yields all listens from the archive that are newer than the given timestamp. +// The listens are yielded in ascending order of their listened_at timestamp. +func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { + return func(yield func(Listen, error) bool) { + files, err := a.ListListenExports() + if err != nil { + yield(Listen{}, err) + return + } + + sort.Slice(files, func(i, j int) bool { + return files[i].TimeRange.Start.Before(files[j].TimeRange.Start) + }) + + for _, file := range files { + if file.TimeRange.End.Before(minTimestamp) { + continue + } + + f := JSONLFile[Listen]{file: file.f} + for l, err := range f.IterItems() { + if err != nil { + yield(Listen{}, err) + return + } + + if !time.Unix(l.ListenedAt, 0).After(minTimestamp) { + continue + } + if !yield(l, nil) { + break + } + } + } + } +} + +// Yields all feedbacks from the archive that are newer than the given timestamp. +// The feedbacks are yielded in ascending order of their Created timestamp. +func (a *ExportArchive) IterFeedback(minTimestamp time.Time) iter.Seq2[Feedback, error] { + return func(yield func(Feedback, error) bool) { + files, err := a.backend.Glob("feedback.jsonl") + if err != nil { + yield(Feedback{}, err) + return + } else if len(files) == 0 { + yield(Feedback{}, errors.New("no feedback.jsonl file found in archive")) + return + } + + j := JSONLFile[Feedback]{file: files[0].File} + for l, err := range j.IterItems() { + if err != nil { + yield(Feedback{}, err) + return + } + + if !time.Unix(l.Created, 0).After(minTimestamp) { + continue + } + if !yield(l, nil) { + break + } + } + } +} + +type UserInfo struct { + ID string `json:"user_id"` + Name string `json:"username"` +} + +type timeRange struct { + Start time.Time + End time.Time +} + +type ListenExportFileInfo struct { + Name string + TimeRange timeRange + f archive.OpenableFile +} + +type JSONLFile[T any] struct { + file archive.OpenableFile +} + +func (f *JSONLFile[T]) openReader() (*jsonl.Reader, error) { + fio, err := f.file.Open() + if err != nil { + return nil, err + } + reader := jsonl.NewReader(fio) + return &reader, nil +} + +func (f *JSONLFile[T]) IterItems() iter.Seq2[T, error] { + return func(yield func(T, error) bool) { + reader, err := f.openReader() + if err != nil { + var listen T + yield(listen, err) + return + } + defer reader.Close() + + for { + var listen T + err := reader.ReadSingleLine(&listen) + if err != nil { + break + } + if !yield(listen, nil) { + break + } + } + } +} + +func getMonthTimeRange(year string, month string) (*timeRange, error) { + yearInt, err := strconv.Atoi(year) + if err != nil { + return nil, err + } + monthInt, err := strconv.Atoi(month) + if err != nil { + return nil, err + } + + r := &timeRange{} + r.Start = time.Date(yearInt, time.Month(monthInt), 1, 0, 0, 0, 0, time.UTC) + + // Get the end of the month + nextMonth := monthInt + 1 + r.End = time.Date( + yearInt, time.Month(nextMonth), 1, 0, 0, 0, 0, time.UTC).Add(-time.Second) + return r, nil +} diff --git a/internal/backends/listenbrainz/client.go b/internal/listenbrainz/client.go similarity index 96% rename from internal/backends/listenbrainz/client.go rename to internal/listenbrainz/client.go index d1a1fa6..957a946 100644 --- a/internal/backends/listenbrainz/client.go +++ b/internal/listenbrainz/client.go @@ -28,7 +28,6 @@ import ( "time" "github.com/go-resty/resty/v2" - "go.uploadedlobster.com/scotty/internal/version" "go.uploadedlobster.com/scotty/pkg/ratelimit" ) @@ -44,13 +43,13 @@ type Client struct { MaxResults int } -func NewClient(token string) Client { +func NewClient(token string, userAgent string) Client { client := resty.New() client.SetBaseURL(listenBrainzBaseURL) client.SetAuthScheme("Token") client.SetAuthToken(token) client.SetHeader("Accept", "application/json") - client.SetHeader("User-Agent", version.UserAgent()) + client.SetHeader("User-Agent", userAgent) // Handle rate limiting (see https://listenbrainz.readthedocs.io/en/latest/users/api/index.html#rate-limiting) ratelimit.EnableHTTPHeaderRateLimit(client, "X-RateLimit-Reset-In") diff --git a/internal/backends/listenbrainz/client_test.go b/internal/listenbrainz/client_test.go similarity index 92% rename from internal/backends/listenbrainz/client_test.go rename to internal/listenbrainz/client_test.go index 45bb0de..9baf293 100644 --- a/internal/backends/listenbrainz/client_test.go +++ b/internal/listenbrainz/client_test.go @@ -31,12 +31,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" + "go.uploadedlobster.com/scotty/internal/listenbrainz" ) func TestNewClient(t *testing.T) { token := "foobar123" - client := listenbrainz.NewClient(token) + client := listenbrainz.NewClient(token, "test/1.0") assert.Equal(t, token, client.HTTPClient.Token) assert.Equal(t, listenbrainz.DefaultItemsPerGet, client.MaxResults) } @@ -44,7 +44,7 @@ func TestNewClient(t *testing.T) { func TestGetListens(t *testing.T) { defer httpmock.DeactivateAndReset() - client := listenbrainz.NewClient("thetoken") + client := listenbrainz.NewClient("thetoken", "test/1.0") client.MaxResults = 2 setupHTTPMock(t, client.HTTPClient.GetClient(), "https://api.listenbrainz.org/1/user/outsidecontext/listens", @@ -64,7 +64,7 @@ func TestGetListens(t *testing.T) { } func TestSubmitListens(t *testing.T) { - client := listenbrainz.NewClient("thetoken") + client := listenbrainz.NewClient("thetoken", "test/1.0") httpmock.ActivateNonDefault(client.HTTPClient.GetClient()) responder, err := httpmock.NewJsonResponder(200, listenbrainz.StatusResult{ @@ -104,7 +104,7 @@ func TestSubmitListens(t *testing.T) { func TestGetFeedback(t *testing.T) { defer httpmock.DeactivateAndReset() - client := listenbrainz.NewClient("thetoken") + client := listenbrainz.NewClient("thetoken", "test/1.0") client.MaxResults = 2 setupHTTPMock(t, client.HTTPClient.GetClient(), "https://api.listenbrainz.org/1/feedback/user/outsidecontext/get-feedback", @@ -123,7 +123,7 @@ func TestGetFeedback(t *testing.T) { } func TestSendFeedback(t *testing.T) { - client := listenbrainz.NewClient("thetoken") + client := listenbrainz.NewClient("thetoken", "test/1.0") httpmock.ActivateNonDefault(client.HTTPClient.GetClient()) responder, err := httpmock.NewJsonResponder(200, listenbrainz.StatusResult{ @@ -149,7 +149,7 @@ func TestSendFeedback(t *testing.T) { func TestLookup(t *testing.T) { defer httpmock.DeactivateAndReset() - client := listenbrainz.NewClient("thetoken") + client := listenbrainz.NewClient("thetoken", "test/1.0") setupHTTPMock(t, client.HTTPClient.GetClient(), "https://api.listenbrainz.org/1/metadata/lookup", "testdata/lookup.json") diff --git a/internal/backends/listenbrainz/models.go b/internal/listenbrainz/models.go similarity index 88% rename from internal/backends/listenbrainz/models.go rename to internal/listenbrainz/models.go index ada75d3..0b5f439 100644 --- a/internal/backends/listenbrainz/models.go +++ b/internal/listenbrainz/models.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -55,27 +55,30 @@ type ListenSubmission struct { } type Listen struct { - InsertedAt int64 `json:"inserted_at,omitempty"` - ListenedAt int64 `json:"listened_at"` - RecordingMSID string `json:"recording_msid,omitempty"` - UserName string `json:"user_name,omitempty"` - TrackMetadata Track `json:"track_metadata"` + InsertedAt float64 `json:"inserted_at,omitempty"` + ListenedAt int64 `json:"listened_at"` + RecordingMSID string `json:"recording_msid,omitempty"` + UserName string `json:"user_name,omitempty"` + TrackMetadata Track `json:"track_metadata"` } type Track struct { TrackName string `json:"track_name,omitempty"` ArtistName string `json:"artist_name,omitempty"` ReleaseName string `json:"release_name,omitempty"` + RecordingMSID string `json:"recording_msid,omitempty"` AdditionalInfo map[string]any `json:"additional_info,omitempty"` MBIDMapping *MBIDMapping `json:"mbid_mapping,omitempty"` } type MBIDMapping struct { - RecordingName string `json:"recording_name,omitempty"` - RecordingMBID mbtypes.MBID `json:"recording_mbid,omitempty"` - ReleaseMBID mbtypes.MBID `json:"release_mbid,omitempty"` - ArtistMBIDs []mbtypes.MBID `json:"artist_mbids,omitempty"` - Artists []Artist `json:"artists,omitempty"` + ArtistMBIDs []mbtypes.MBID `json:"artist_mbids,omitempty"` + Artists []Artist `json:"artists,omitempty"` + RecordingMBID mbtypes.MBID `json:"recording_mbid,omitempty"` + RecordingName string `json:"recording_name,omitempty"` + ReleaseMBID mbtypes.MBID `json:"release_mbid,omitempty"` + CAAID int `json:"caa_id,omitempty"` + CAAReleaseMBID mbtypes.MBID `json:"caa_release_mbid,omitempty"` } type Artist struct { diff --git a/internal/backends/listenbrainz/models_test.go b/internal/listenbrainz/models_test.go similarity index 97% rename from internal/backends/listenbrainz/models_test.go rename to internal/listenbrainz/models_test.go index 02cbe98..404b87b 100644 --- a/internal/backends/listenbrainz/models_test.go +++ b/internal/listenbrainz/models_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" + "go.uploadedlobster.com/scotty/internal/listenbrainz" ) func TestTrackDurationMillisecondsInt(t *testing.T) { diff --git a/internal/backends/listenbrainz/testdata/feedback.json b/internal/listenbrainz/testdata/feedback.json similarity index 100% rename from internal/backends/listenbrainz/testdata/feedback.json rename to internal/listenbrainz/testdata/feedback.json diff --git a/internal/backends/listenbrainz/testdata/listen.json b/internal/listenbrainz/testdata/listen.json similarity index 100% rename from internal/backends/listenbrainz/testdata/listen.json rename to internal/listenbrainz/testdata/listen.json diff --git a/internal/backends/listenbrainz/testdata/listens.json b/internal/listenbrainz/testdata/listens.json similarity index 100% rename from internal/backends/listenbrainz/testdata/listens.json rename to internal/listenbrainz/testdata/listens.json diff --git a/internal/backends/listenbrainz/testdata/lookup.json b/internal/listenbrainz/testdata/lookup.json similarity index 100% rename from internal/backends/listenbrainz/testdata/lookup.json rename to internal/listenbrainz/testdata/lookup.json