Simplify dirArchive by using os.dirFS and have Archive.Open return fs.File

README.md

@@ -117,18 +117,19 @@ scotty beam listens deezer listenbrainz --timestamp "2023-12-06 14:26:24"
 ### Supported backends
 The following table lists the available backends and the currently supported features.
 
-Backend         | Listens Export | Listens Import | Loves Export | Loves Import
+Backend              | Listens Export | Listens Import | Loves Export | Loves Import
-----------------|----------------|----------------|--------------|-------------
+---------------------|----------------|----------------|--------------|-------------
-deezer          | ✓              | ⨯              | ✓            | -
+deezer               | ✓              | ⨯              | ✓            | -
-funkwhale       | ✓              | ⨯              | ✓            | -
+funkwhale            | ✓              | ⨯              | ✓            | -
-jspf            | ✓              | ✓              | ✓            | ✓
+jspf                 | ✓              | ✓              | ✓            | ✓
-lastfm          | ✓              | ✓              | ✓            | ✓
+lastfm               | ✓              | ✓              | ✓            | ✓
-listenbrainz    | ✓              | ✓              | ✓            | ✓
+listenbrainz         | ✓              | ✓              | ✓            | ✓
-maloja          | ✓              | ✓              | ⨯            | ⨯
+listenbrainz-archive | ✓              | -              | -            | -
-scrobbler-log   | ✓              | ✓              | ⨯            | ⨯
+maloja               | ✓              | ✓              | ⨯            | ⨯
-spotify         | ✓              | ⨯              | ✓            | -
+scrobbler-log        | ✓              | ✓              | ⨯            | ⨯
-spotify-history | ✓              | ⨯              | ⨯            | ⨯
+spotify              | ✓              | ⨯              | ✓            | -
-subsonic        | ⨯              | ⨯              | ✓            | -
+spotify-history      | ✓              | ⨯              | ⨯            | ⨯
+subsonic             | ⨯              | ⨯              | ✓            | -
 
 **✓** implemented **-** not yet implemented **⨯** unavailable / not planned
 

config.example.toml

@@ -19,6 +19,13 @@ token    = ""
 # not already exists in your ListenBrainz profile.
 check-duplicate-listens = false
 
+[service.listenbrainz-archive]
+# This backend supports listens from a ListenBrainz export archive
+# (https://listenbrainz.org/settings/export/).
+backend   = "listenbrainz-archive"
+# The file path to the ListenBrainz export archive.
+file-path = "./listenbrainz_outsidecontext.zip"
+
 [service.maloja]
 # Maloja is a self hosted listening service (https://github.com/krateng/maloja)
 backend    = "maloja"

go.mod

@@ -53,6 +53,7 @@ require (
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.9.0 // indirect
+	github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.14.0 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect

go.sum

@@ -107,6 +107,8 @@ github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFT
 github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk=
 github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0 h1:cgqwZtnR+IQfUYDLJ3Kiy4aE+O/wExTzEIg8xwC4Qfs=
 github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0/go.mod h1:n3nudMl178cEvD44PaopxH9jhJaQzthSxUzLO5iKMy4=
+github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 h1:CXJI+lliMiiEwzfgE8yt/38K0heYDgQ0L3f/3fxRnQU=
+github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740/go.mod h1:G4w16caPmc6at7u4fmkj/8OAoOnM9mkmJr2fvL0vhaw=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
 github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
 github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA=

internal/archive/archive.go

@@ -0,0 +1,179 @@
+/*
+Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+// Implements generic access to files inside an archive.
+//
+// An archive in this context can be any container that holds files.
+// In this implementation the archive can be a ZIP file or a directory.
+package archive
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+// Generic archive interface.
+type Archive interface {
+	Close() error
+	Open(path string) (fs.File, error)
+	Glob(pattern string) ([]FileInfo, error)
+}
+
+// Open an archive in path.
+// The archive can be a ZIP file or a directory. The implementation
+// will detect the type of archive and return the appropriate
+// implementation of the Archive interface.
+func OpenArchive(path string) (Archive, error) {
+	fi, err := os.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+	switch mode := fi.Mode(); {
+	case mode.IsRegular():
+		archive := &zipArchive{}
+		err := archive.OpenArchive(path)
+		if err != nil {
+			return nil, err
+		}
+		return archive, nil
+	case mode.IsDir():
+		archive := &dirArchive{}
+		err := archive.OpenArchive(path)
+		if err != nil {
+			return nil, err
+		}
+		return archive, nil
+	default:
+		return nil, fmt.Errorf("unsupported file mode: %s", mode)
+	}
+}
+
+// Interface for a file that can be opened when needed.
+type OpenableFile interface {
+	Open() (io.ReadCloser, error)
+}
+
+// Generic information about a file inside an archive.
+type FileInfo struct {
+	Name string
+	File OpenableFile
+}
+
+// A openable file in the filesystem.
+type filesystemFile struct {
+	path string
+}
+
+func (f *filesystemFile) Open() (io.ReadCloser, error) {
+	return os.Open(f.path)
+}
+
+// An implementation of the archiveBackend interface for zip files.
+type zipArchive struct {
+	zip *zip.ReadCloser
+}
+
+func (a *zipArchive) OpenArchive(path string) error {
+	zip, err := zip.OpenReader(path)
+	if err != nil {
+		return err
+	}
+	a.zip = zip
+	return nil
+}
+
+func (a *zipArchive) Close() error {
+	if a.zip == nil {
+		return nil
+	}
+	return a.zip.Close()
+}
+
+func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) {
+	result := make([]FileInfo, 0)
+	for _, file := range a.zip.File {
+		if matched, err := filepath.Match(pattern, file.Name); matched {
+			if err != nil {
+				return nil, err
+			}
+			info := FileInfo{
+				Name: file.Name,
+				File: file,
+			}
+			result = append(result, info)
+		}
+	}
+
+	return result, nil
+}
+
+func (a *zipArchive) Open(path string) (fs.File, error) {
+	file, err := a.zip.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+// An implementation of the archiveBackend interface for directories.
+type dirArchive struct {
+	path  string
+	dirFS fs.FS
+}
+
+func (a *dirArchive) OpenArchive(path string) error {
+	a.path = filepath.Clean(path)
+	a.dirFS = os.DirFS(path)
+	return nil
+}
+
+func (a *dirArchive) Close() error {
+	return nil
+}
+
+// Open opens the named file in the archive.
+// [fs.File.Close] must be called to release any associated resources.
+func (a *dirArchive) Open(path string) (fs.File, error) {
+	return a.dirFS.Open(path)
+}
+
+func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) {
+	files, err := fs.Glob(a.dirFS, pattern)
+	if err != nil {
+		return nil, err
+	}
+	result := make([]FileInfo, 0)
+	for _, name := range files {
+		fullPath := filepath.Join(a.path, name)
+		info := FileInfo{
+			Name: name,
+			File: &filesystemFile{path: fullPath},
+		}
+		result = append(result, info)
+	}
+
+	return result, nil
+}

internal/backends/backends.go

@@ -27,6 +27,7 @@ import (
 	"go.uploadedlobster.com/scotty/internal/backends/funkwhale"
 	"go.uploadedlobster.com/scotty/internal/backends/jspf"
 	"go.uploadedlobster.com/scotty/internal/backends/lastfm"
+	"go.uploadedlobster.com/scotty/internal/backends/lbarchive"
 	"go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
 	"go.uploadedlobster.com/scotty/internal/backends/maloja"
 	"go.uploadedlobster.com/scotty/internal/backends/scrobblerlog"
@@ -105,17 +106,18 @@ func GetBackends() BackendList {
 }
 
 var knownBackends = map[string]func() models.Backend{
-	"deezer":          func() models.Backend { return &deezer.DeezerApiBackend{} },
+	"deezer":               func() models.Backend { return &deezer.DeezerApiBackend{} },
-	"dump":            func() models.Backend { return &dump.DumpBackend{} },
+	"dump":                 func() models.Backend { return &dump.DumpBackend{} },
-	"funkwhale":       func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} },
+	"funkwhale":            func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} },
-	"jspf":            func() models.Backend { return &jspf.JSPFBackend{} },
+	"jspf":                 func() models.Backend { return &jspf.JSPFBackend{} },
-	"lastfm":          func() models.Backend { return &lastfm.LastfmApiBackend{} },
+	"lastfm":               func() models.Backend { return &lastfm.LastfmApiBackend{} },
-	"listenbrainz":    func() models.Backend { return &listenbrainz.ListenBrainzApiBackend{} },
+	"listenbrainz":         func() models.Backend { return &listenbrainz.ListenBrainzApiBackend{} },
-	"maloja":          func() models.Backend { return &maloja.MalojaApiBackend{} },
+	"listenbrainz-archive": func() models.Backend { return &lbarchive.ListenBrainzArchiveBackend{} },
-	"scrobbler-log":   func() models.Backend { return &scrobblerlog.ScrobblerLogBackend{} },
+	"maloja":               func() models.Backend { return &maloja.MalojaApiBackend{} },
-	"spotify":         func() models.Backend { return &spotify.SpotifyApiBackend{} },
+	"scrobbler-log":        func() models.Backend { return &scrobblerlog.ScrobblerLogBackend{} },
-	"spotify-history": func() models.Backend { return &spotifyhistory.SpotifyHistoryBackend{} },
+	"spotify":              func() models.Backend { return &spotify.SpotifyApiBackend{} },
-	"subsonic":        func() models.Backend { return &subsonic.SubsonicApiBackend{} },
+	"spotify-history":      func() models.Backend { return &spotifyhistory.SpotifyHistoryBackend{} },
+	"subsonic":             func() models.Backend { return &subsonic.SubsonicApiBackend{} },
 }
 
 func backendWithConfig(config config.ServiceConfig) (models.Backend, error) {

internal/backends/backends_test.go

@@ -28,6 +28,7 @@ import (
 	"go.uploadedlobster.com/scotty/internal/backends/funkwhale"
 	"go.uploadedlobster.com/scotty/internal/backends/jspf"
 	"go.uploadedlobster.com/scotty/internal/backends/lastfm"
+	"go.uploadedlobster.com/scotty/internal/backends/lbarchive"
 	"go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
 	"go.uploadedlobster.com/scotty/internal/backends/maloja"
 	"go.uploadedlobster.com/scotty/internal/backends/scrobblerlog"
@@ -103,6 +104,11 @@ func TestImplementsInterfaces(t *testing.T) {
 	expectInterface[models.LovesExport](t, &lastfm.LastfmApiBackend{})
 	expectInterface[models.LovesImport](t, &lastfm.LastfmApiBackend{})
 
+	expectInterface[models.ListensExport](t, &lbarchive.ListenBrainzArchiveBackend{})
+	// expectInterface[models.ListensImport](t, &lbarchive.ListenBrainzArchiveBackend{})
+	// expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{})
+	// expectInterface[models.LovesImport](t, &lbarchive.ListenBrainzArchiveBackend{})
+
 	expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{})
 	expectInterface[models.ListensImport](t, &listenbrainz.ListenBrainzApiBackend{})
 	expectInterface[models.LovesExport](t, &listenbrainz.ListenBrainzApiBackend{})

internal/backends/lbarchive/lbarchive.go

@@ -0,0 +1,121 @@
+/*
+Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package lbarchive
+
+import (
+	"context"
+	"time"
+
+	lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
+	"go.uploadedlobster.com/scotty/internal/config"
+	"go.uploadedlobster.com/scotty/internal/i18n"
+	"go.uploadedlobster.com/scotty/internal/listenbrainz"
+	"go.uploadedlobster.com/scotty/internal/models"
+)
+
+const batchSize = 2000
+
+type ListenBrainzArchiveBackend struct {
+	filePath string
+}
+
+func (b *ListenBrainzArchiveBackend) Name() string { return "listenbrainz-archive" }
+
+func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption {
+	return []models.BackendOption{{
+		Name:  "file-path",
+		Label: i18n.Tr("Export ZIP file path"),
+		Type:  models.String,
+	}}
+}
+
+func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error {
+	b.filePath = config.GetString("file-path")
+	return nil
+}
+
+func (b *ListenBrainzArchiveBackend) ExportListens(
+	ctx context.Context, oldestTimestamp time.Time,
+	results chan models.ListensResult, progress chan models.TransferProgress) {
+	startTime := time.Now()
+	minTime := oldestTimestamp
+	if minTime.Unix() < 1 {
+		minTime = time.Unix(1, 0)
+	}
+
+	totalDuration := startTime.Sub(oldestTimestamp)
+	p := models.TransferProgress{
+		Export: &models.Progress{
+			Total: int64(totalDuration.Seconds()),
+		},
+	}
+
+	archive, err := listenbrainz.OpenExportArchive(b.filePath)
+	if err != nil {
+		p.Export.Abort()
+		progress <- p
+		results <- models.ListensResult{Error: err}
+		return
+	}
+	defer archive.Close()
+
+	userInfo, err := archive.UserInfo()
+	if err != nil {
+		p.Export.Abort()
+		progress <- p
+		results <- models.ListensResult{Error: err}
+		return
+	}
+
+	listens := make(models.ListensList, 0, batchSize)
+	for rawListen, err := range archive.IterListens(oldestTimestamp) {
+		if err != nil {
+			p.Export.Abort()
+			progress <- p
+			results <- models.ListensResult{Error: err}
+			return
+		}
+
+		listen := lbapi.AsListen(rawListen)
+		if listen.UserName == "" {
+			listen.UserName = userInfo.Name
+		}
+		listens = append(listens, listen)
+
+		// Update the progress
+		p.Export.TotalItems += 1
+		remainingTime := startTime.Sub(listen.ListenedAt)
+		p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
+
+		// Allow the importer to start processing the listens by
+		// sending them in batches.
+		if len(listens) >= batchSize {
+			results <- models.ListensResult{Items: listens}
+			progress <- p
+			listens = listens[:0]
+		}
+	}
+
+	results <- models.ListensResult{Items: listens}
+	p.Export.Complete()
+	progress <- p
+}

internal/backends/lbarchive/lbarchive_test.go

@@ -0,0 +1,40 @@
+/*
+Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package lbarchive_test
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"go.uploadedlobster.com/scotty/internal/backends/lbarchive"
+	"go.uploadedlobster.com/scotty/internal/config"
+)
+
+func TestInitConfig(t *testing.T) {
+	c := viper.New()
+	c.Set("file-path", "/foo/lbarchive.zip")
+	service := config.NewServiceConfig("test", c)
+	backend := lbarchive.ListenBrainzArchiveBackend{}
+	err := backend.InitConfig(&service)
+	assert.NoError(t, err)
+}

internal/backends/listenbrainz/listenbrainz.go

@@ -26,13 +26,14 @@ import (
 	"go.uploadedlobster.com/musicbrainzws2"
 	"go.uploadedlobster.com/scotty/internal/config"
 	"go.uploadedlobster.com/scotty/internal/i18n"
+	"go.uploadedlobster.com/scotty/internal/listenbrainz"
 	"go.uploadedlobster.com/scotty/internal/models"
 	"go.uploadedlobster.com/scotty/internal/similarity"
 	"go.uploadedlobster.com/scotty/internal/version"
 )
 
 type ListenBrainzApiBackend struct {
-	client          Client
+	client          listenbrainz.Client
 	mbClient        musicbrainzws2.Client
 	username        string
 	checkDuplicates bool
@@ -58,13 +59,13 @@ func (b *ListenBrainzApiBackend) Options() []models.BackendOption {
 }
 
 func (b *ListenBrainzApiBackend) InitConfig(config *config.ServiceConfig) error {
-	b.client = NewClient(config.GetString("token"))
+	b.client = listenbrainz.NewClient(config.GetString("token"), version.UserAgent())
 	b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{
 		Name:    version.AppName,
 		Version: version.AppVersion,
 		URL:     version.AppURL,
 	})
-	b.client.MaxResults = MaxItemsPerGet
+	b.client.MaxResults = listenbrainz.MaxItemsPerGet
 	b.username = config.GetString("username")
 	b.checkDuplicates = config.GetBool("check-duplicate-listens", false)
 	return nil
@@ -116,7 +117,7 @@ func (b *ListenBrainzApiBackend) ExportListens(ctx context.Context, oldestTimest
 
 		for _, listen := range result.Payload.Listens {
 			if listen.ListenedAt > oldestTimestamp.Unix() {
-				listens = append(listens, listen.AsListen())
+				listens = append(listens, AsListen(listen))
 			} else {
 				// result contains listens older then oldestTimestamp
 				break
@@ -138,16 +139,16 @@ func (b *ListenBrainzApiBackend) ExportListens(ctx context.Context, oldestTimest
 func (b *ListenBrainzApiBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
 	total := len(export.Items)
 	p := models.TransferProgress{}.FromImportResult(importResult, false)
-	for i := 0; i < total; i += MaxListensPerRequest {
+	for i := 0; i < total; i += listenbrainz.MaxListensPerRequest {
-		listens := export.Items[i:min(i+MaxListensPerRequest, total)]
+		listens := export.Items[i:min(i+listenbrainz.MaxListensPerRequest, total)]
 		count := len(listens)
 		if count == 0 {
 			break
 		}
 
-		submission := ListenSubmission{
+		submission := listenbrainz.ListenSubmission{
-			ListenType: Import,
+			ListenType: listenbrainz.Import,
-			Payload:    make([]Listen, 0, count),
+			Payload:    make([]listenbrainz.Listen, 0, count),
 		}
 
 		for _, l := range listens {
@@ -167,9 +168,9 @@ func (b *ListenBrainzApiBackend) ImportListens(ctx context.Context, export model
 			}
 
 			l.FillAdditionalInfo()
-			listen := Listen{
+			listen := listenbrainz.Listen{
 				ListenedAt: l.ListenedAt.Unix(),
-				TrackMetadata: Track{
+				TrackMetadata: listenbrainz.Track{
 					TrackName:      l.TrackName,
 					ReleaseName:    l.ReleaseName,
 					ArtistName:     l.ArtistName(),
@@ -228,7 +229,7 @@ func (b *ListenBrainzApiBackend) ExportLoves(ctx context.Context, oldestTimestam
 func (b *ListenBrainzApiBackend) exportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult) {
 	offset := 0
 	defer close(results)
-	loves := make(models.LovesList, 0, 2*MaxItemsPerGet)
+	loves := make(models.LovesList, 0, 2*listenbrainz.MaxItemsPerGet)
 
 out:
 	for {
@@ -254,7 +255,7 @@ out:
 				}
 			}
 
-			love := feedback.AsLove()
+			love := AsLove(feedback)
 			if love.Created.After(oldestTimestamp) {
 				loves = append(loves, love)
 			} else {
@@ -262,7 +263,7 @@ out:
 			}
 		}
 
-		offset += MaxItemsPerGet
+		offset += listenbrainz.MaxItemsPerGet
 	}
 
 	sort.Sort(loves)
@@ -278,7 +279,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(ctx context.Context, export models.
 		go b.exportLoves(ctx, time.Unix(0, 0), existingLovesChan)
 
 		// TODO: Store MBIDs directly
-		b.existingMBIDs = make(map[mbtypes.MBID]bool, MaxItemsPerGet)
+		b.existingMBIDs = make(map[mbtypes.MBID]bool, listenbrainz.MaxItemsPerGet)
 
 		for existingLoves := range existingLovesChan {
 			if existingLoves.Error != nil {
@@ -316,7 +317,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(ctx context.Context, export models.
 			if b.existingMBIDs[recordingMBID] {
 				ok = true
 			} else {
-				resp, err := b.client.SendFeedback(ctx, Feedback{
+				resp, err := b.client.SendFeedback(ctx, listenbrainz.Feedback{
 					RecordingMBID: recordingMBID,
 					Score:         1,
 				})
@@ -366,7 +367,7 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste
 	}
 
 	for _, c := range candidates.Payload.Listens {
-		sim := similarity.CompareTracks(listen.Track, c.TrackMetadata.AsTrack())
+		sim := similarity.CompareTracks(listen.Track, AsTrack(c.TrackMetadata))
 		if sim >= trackSimilarityThreshold {
 			return true, nil
 		}
@@ -375,7 +376,8 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste
 	return false, nil
 }
 
-func (b *ListenBrainzApiBackend) lookupRecording(ctx context.Context, mbid mbtypes.MBID) (*Track, error) {
+func (b *ListenBrainzApiBackend) lookupRecording(
+	ctx context.Context, mbid mbtypes.MBID) (*listenbrainz.Track, error) {
 	filter := musicbrainzws2.IncludesFilter{
 		Includes: []string{"artist-credits"},
 	}
@@ -388,10 +390,10 @@ func (b *ListenBrainzApiBackend) lookupRecording(ctx context.Context, mbid mbtyp
 	for _, artist := range recording.ArtistCredit {
 		artistMBIDs = append(artistMBIDs, artist.Artist.ID)
 	}
-	track := Track{
+	track := listenbrainz.Track{
 		TrackName:  recording.Title,
 		ArtistName: recording.ArtistCredit.String(),
-		MBIDMapping: &MBIDMapping{
+		MBIDMapping: &listenbrainz.MBIDMapping{
 			// In case of redirects this MBID differs from the looked up MBID
 			RecordingMBID: recording.ID,
 			ArtistMBIDs:   artistMBIDs,
@@ -400,26 +402,26 @@ func (b *ListenBrainzApiBackend) lookupRecording(ctx context.Context, mbid mbtyp
 	return &track, nil
 }
 
-func (lbListen Listen) AsListen() models.Listen {
+func AsListen(lbListen listenbrainz.Listen) models.Listen {
 	listen := models.Listen{
 		ListenedAt: time.Unix(lbListen.ListenedAt, 0),
 		UserName:   lbListen.UserName,
-		Track:      lbListen.TrackMetadata.AsTrack(),
+		Track:      AsTrack(lbListen.TrackMetadata),
 	}
 	return listen
 }
 
-func (f Feedback) AsLove() models.Love {
+func AsLove(f listenbrainz.Feedback) models.Love {
 	recordingMBID := f.RecordingMBID
 	track := f.TrackMetadata
 	if track == nil {
-		track = &Track{}
+		track = &listenbrainz.Track{}
 	}
 	love := models.Love{
 		UserName:      f.UserName,
 		RecordingMBID: recordingMBID,
 		Created:       time.Unix(f.Created, 0),
-		Track:         track.AsTrack(),
+		Track:         AsTrack(*track),
 	}
 
 	if love.Track.RecordingMBID == "" {
@@ -429,7 +431,7 @@ func (f Feedback) AsLove() models.Love {
 	return love
 }
 
-func (t Track) AsTrack() models.Track {
+func AsTrack(t listenbrainz.Track) models.Track {
 	track := models.Track{
 		TrackName:        t.TrackName,
 		ReleaseName:      t.ReleaseName,

internal/backends/listenbrainz/listenbrainz_test.go

@@ -24,15 +24,16 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uploadedlobster.com/mbtypes"
-	"go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
+	lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
 	"go.uploadedlobster.com/scotty/internal/config"
+	"go.uploadedlobster.com/scotty/internal/listenbrainz"
 )
 
 func TestInitConfig(t *testing.T) {
 	c := viper.New()
 	c.Set("token", "thetoken")
 	service := config.NewServiceConfig("test", c)
-	backend := listenbrainz.ListenBrainzApiBackend{}
+	backend := lbapi.ListenBrainzApiBackend{}
 	err := backend.InitConfig(&service)
 	assert.NoError(t, err)
 }
@@ -57,7 +58,7 @@ func TestListenBrainzListenAsListen(t *testing.T) {
 			},
 		},
 	}
-	listen := lbListen.AsListen()
+	listen := lbapi.AsListen(lbListen)
 	assert.Equal(t, time.Unix(1699289873, 0), listen.ListenedAt)
 	assert.Equal(t, lbListen.UserName, listen.UserName)
 	assert.Equal(t, time.Duration(413787*time.Millisecond), listen.Duration)
@@ -93,7 +94,7 @@ func TestListenBrainzFeedbackAsLove(t *testing.T) {
 			},
 		},
 	}
-	love := feedback.AsLove()
+	love := lbapi.AsLove(feedback)
 	assert := assert.New(t)
 	assert.Equal(time.Unix(1699859066, 0).Unix(), love.Created.Unix())
 	assert.Equal(feedback.UserName, love.UserName)
@@ -114,7 +115,7 @@ func TestListenBrainzPartialFeedbackAsLove(t *testing.T) {
 		RecordingMBID: recordingMBID,
 		Score:         1,
 	}
-	love := feedback.AsLove()
+	love := lbapi.AsLove(feedback)
 	assert := assert.New(t)
 	assert.Equal(time.Unix(1699859066, 0).Unix(), love.Created.Unix())
 	assert.Equal(recordingMBID, love.RecordingMBID)

internal/backends/spotifyhistory/spotifyhistory.go

@@ -20,7 +20,6 @@ package spotifyhistory
 import (
 	"context"
 	"os"
-	"path"
 	"path/filepath"
 	"slices"
 	"sort"
@@ -74,7 +73,7 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error {
 }
 
 func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
-	files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob))
+	files, err := filepath.Glob(filepath.Join(b.dirPath, historyFileGlob))
 	p := models.TransferProgress{
 		Export: &models.Progress{},
 	}

internal/config/config.go

@@ -19,7 +19,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -40,7 +39,7 @@ const (
 func DefaultConfigDir() string {
 	configDir, err := os.UserConfigDir()
 	cobra.CheckErr(err)
-	return path.Join(configDir, version.AppName)
+	return filepath.Join(configDir, version.AppName)
 }
 
 // initConfig reads in config file and ENV variables if set.

internal/listenbrainz/archive.go

@@ -0,0 +1,251 @@
+/*
+Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package listenbrainz
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"iter"
+	"regexp"
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/simonfrey/jsonl"
+	"go.uploadedlobster.com/scotty/internal/archive"
+)
+
+// Represents a ListenBrainz export archive.
+//
+// The export contains the user's listen history, favorite tracks and
+// user information.
+type ExportArchive struct {
+	backend archive.Archive
+}
+
+// Open a ListenBrainz archive from file path.
+func OpenExportArchive(path string) (*ExportArchive, error) {
+	backend, err := archive.OpenArchive(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ExportArchive{backend: backend}, nil
+}
+
+// Close the archive and release any resources.
+func (a *ExportArchive) Close() error {
+	if a.backend == nil {
+		return nil
+	}
+	return a.backend.Close()
+}
+
+// Read the user information from the archive.
+func (a *ExportArchive) UserInfo() (UserInfo, error) {
+	f, err := a.backend.Open("user.json")
+	if err != nil {
+		return UserInfo{}, err
+	}
+	defer f.Close()
+
+	userInfo := UserInfo{}
+	bytes, err := io.ReadAll(f)
+	if err != nil {
+		return userInfo, err
+	}
+
+	json.Unmarshal(bytes, &userInfo)
+	return userInfo, nil
+}
+
+func (a *ExportArchive) ListListenExports() ([]ListenExportFileInfo, error) {
+	re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`)
+	result := make([]ListenExportFileInfo, 0)
+
+	files, err := a.backend.Glob("listens/*/*.jsonl")
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		match := re.FindStringSubmatch(file.Name)
+		if match == nil {
+			continue
+		}
+
+		year := match[1]
+		month := match[2]
+		times, err := getMonthTimeRange(year, month)
+		if err != nil {
+			return nil, err
+		}
+		info := ListenExportFileInfo{
+			Name:      file.Name,
+			TimeRange: *times,
+			f:         file.File,
+		}
+		result = append(result, info)
+	}
+
+	return result, nil
+}
+
+// Yields all listens from the archive that are newer than the given timestamp.
+// The listens are yielded in ascending order of their listened_at timestamp.
+func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] {
+	return func(yield func(Listen, error) bool) {
+		files, err := a.ListListenExports()
+		if err != nil {
+			yield(Listen{}, err)
+			return
+		}
+
+		sort.Slice(files, func(i, j int) bool {
+			return files[i].TimeRange.Start.Before(files[j].TimeRange.Start)
+		})
+
+		for _, file := range files {
+			if file.TimeRange.End.Before(minTimestamp) {
+				continue
+			}
+
+			f := JSONLFile[Listen]{file: file.f}
+			for l, err := range f.IterItems() {
+				if err != nil {
+					yield(Listen{}, err)
+					return
+				}
+
+				if !time.Unix(l.ListenedAt, 0).After(minTimestamp) {
+					continue
+				}
+				if !yield(l, nil) {
+					break
+				}
+			}
+		}
+	}
+}
+
+// Yields all feedbacks from the archive that are newer than the given timestamp.
+// The feedbacks are yielded in ascending order of their Created timestamp.
+func (a *ExportArchive) IterFeedback(minTimestamp time.Time) iter.Seq2[Feedback, error] {
+	return func(yield func(Feedback, error) bool) {
+		files, err := a.backend.Glob("feedback.jsonl")
+		if err != nil {
+			yield(Feedback{}, err)
+			return
+		} else if len(files) == 0 {
+			yield(Feedback{}, errors.New("no feedback.jsonl file found in archive"))
+			return
+		}
+
+		j := JSONLFile[Feedback]{file: files[0].File}
+		for l, err := range j.IterItems() {
+			if err != nil {
+				yield(Feedback{}, err)
+				return
+			}
+
+			if !time.Unix(l.Created, 0).After(minTimestamp) {
+				continue
+			}
+			if !yield(l, nil) {
+				break
+			}
+		}
+	}
+}
+
+type UserInfo struct {
+	ID   string `json:"user_id"`
+	Name string `json:"username"`
+}
+
+type timeRange struct {
+	Start time.Time
+	End   time.Time
+}
+
+type ListenExportFileInfo struct {
+	Name      string
+	TimeRange timeRange
+	f         archive.OpenableFile
+}
+
+type JSONLFile[T any] struct {
+	file archive.OpenableFile
+}
+
+func (f *JSONLFile[T]) openReader() (*jsonl.Reader, error) {
+	fio, err := f.file.Open()
+	if err != nil {
+		return nil, err
+	}
+	reader := jsonl.NewReader(fio)
+	return &reader, nil
+}
+
+func (f *JSONLFile[T]) IterItems() iter.Seq2[T, error] {
+	return func(yield func(T, error) bool) {
+		reader, err := f.openReader()
+		if err != nil {
+			var listen T
+			yield(listen, err)
+			return
+		}
+		defer reader.Close()
+
+		for {
+			var listen T
+			err := reader.ReadSingleLine(&listen)
+			if err != nil {
+				break
+			}
+			if !yield(listen, nil) {
+				break
+			}
+		}
+	}
+}
+
+func getMonthTimeRange(year string, month string) (*timeRange, error) {
+	yearInt, err := strconv.Atoi(year)
+	if err != nil {
+		return nil, err
+	}
+	monthInt, err := strconv.Atoi(month)
+	if err != nil {
+		return nil, err
+	}
+
+	r := &timeRange{}
+	r.Start = time.Date(yearInt, time.Month(monthInt), 1, 0, 0, 0, 0, time.UTC)
+
+	// Get the end of the month
+	nextMonth := monthInt + 1
+	r.End = time.Date(
+		yearInt, time.Month(nextMonth), 1, 0, 0, 0, 0, time.UTC).Add(-time.Second)
+	return r, nil
+}

internal/listenbrainz/client.go

@@ -28,7 +28,6 @@ import (
 	"time"
 
 	"github.com/go-resty/resty/v2"
-	"go.uploadedlobster.com/scotty/internal/version"
 	"go.uploadedlobster.com/scotty/pkg/ratelimit"
 )
 
@@ -44,13 +43,13 @@ type Client struct {
 	MaxResults int
 }
 
-func NewClient(token string) Client {
+func NewClient(token string, userAgent string) Client {
 	client := resty.New()
 	client.SetBaseURL(listenBrainzBaseURL)
 	client.SetAuthScheme("Token")
 	client.SetAuthToken(token)
 	client.SetHeader("Accept", "application/json")
-	client.SetHeader("User-Agent", version.UserAgent())
+	client.SetHeader("User-Agent", userAgent)
 
 	// Handle rate limiting (see https://listenbrainz.readthedocs.io/en/latest/users/api/index.html#rate-limiting)
 	ratelimit.EnableHTTPHeaderRateLimit(client, "X-RateLimit-Reset-In")

internal/listenbrainz/client_test.go

@@ -31,12 +31,12 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uploadedlobster.com/mbtypes"
-	"go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
+	"go.uploadedlobster.com/scotty/internal/listenbrainz"
 )
 
 func TestNewClient(t *testing.T) {
 	token := "foobar123"
-	client := listenbrainz.NewClient(token)
+	client := listenbrainz.NewClient(token, "test/1.0")
 	assert.Equal(t, token, client.HTTPClient.Token)
 	assert.Equal(t, listenbrainz.DefaultItemsPerGet, client.MaxResults)
 }
@@ -44,7 +44,7 @@ func TestNewClient(t *testing.T) {
 func TestGetListens(t *testing.T) {
 	defer httpmock.DeactivateAndReset()
 
-	client := listenbrainz.NewClient("thetoken")
+	client := listenbrainz.NewClient("thetoken", "test/1.0")
 	client.MaxResults = 2
 	setupHTTPMock(t, client.HTTPClient.GetClient(),
 		"https://api.listenbrainz.org/1/user/outsidecontext/listens",
@@ -64,7 +64,7 @@ func TestGetListens(t *testing.T) {
 }
 
 func TestSubmitListens(t *testing.T) {
-	client := listenbrainz.NewClient("thetoken")
+	client := listenbrainz.NewClient("thetoken", "test/1.0")
 	httpmock.ActivateNonDefault(client.HTTPClient.GetClient())
 
 	responder, err := httpmock.NewJsonResponder(200, listenbrainz.StatusResult{
@@ -104,7 +104,7 @@ func TestSubmitListens(t *testing.T) {
 func TestGetFeedback(t *testing.T) {
 	defer httpmock.DeactivateAndReset()
 
-	client := listenbrainz.NewClient("thetoken")
+	client := listenbrainz.NewClient("thetoken", "test/1.0")
 	client.MaxResults = 2
 	setupHTTPMock(t, client.HTTPClient.GetClient(),
 		"https://api.listenbrainz.org/1/feedback/user/outsidecontext/get-feedback",
@@ -123,7 +123,7 @@ func TestGetFeedback(t *testing.T) {
 }
 
 func TestSendFeedback(t *testing.T) {
-	client := listenbrainz.NewClient("thetoken")
+	client := listenbrainz.NewClient("thetoken", "test/1.0")
 	httpmock.ActivateNonDefault(client.HTTPClient.GetClient())
 
 	responder, err := httpmock.NewJsonResponder(200, listenbrainz.StatusResult{
@@ -149,7 +149,7 @@ func TestSendFeedback(t *testing.T) {
 func TestLookup(t *testing.T) {
 	defer httpmock.DeactivateAndReset()
 
-	client := listenbrainz.NewClient("thetoken")
+	client := listenbrainz.NewClient("thetoken", "test/1.0")
 	setupHTTPMock(t, client.HTTPClient.GetClient(),
 		"https://api.listenbrainz.org/1/metadata/lookup",
 		"testdata/lookup.json")

internal/listenbrainz/models.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -55,27 +55,30 @@ type ListenSubmission struct {
 }
 
 type Listen struct {
-	InsertedAt    int64  `json:"inserted_at,omitempty"`
+	InsertedAt    float64 `json:"inserted_at,omitempty"`
-	ListenedAt    int64  `json:"listened_at"`
+	ListenedAt    int64   `json:"listened_at"`
-	RecordingMSID string `json:"recording_msid,omitempty"`
+	RecordingMSID string  `json:"recording_msid,omitempty"`
-	UserName      string `json:"user_name,omitempty"`
+	UserName      string  `json:"user_name,omitempty"`
-	TrackMetadata Track  `json:"track_metadata"`
+	TrackMetadata Track   `json:"track_metadata"`
 }
 
 type Track struct {
 	TrackName      string         `json:"track_name,omitempty"`
 	ArtistName     string         `json:"artist_name,omitempty"`
 	ReleaseName    string         `json:"release_name,omitempty"`
+	RecordingMSID  string         `json:"recording_msid,omitempty"`
 	AdditionalInfo map[string]any `json:"additional_info,omitempty"`
 	MBIDMapping    *MBIDMapping   `json:"mbid_mapping,omitempty"`
 }
 
 type MBIDMapping struct {
-	RecordingName string         `json:"recording_name,omitempty"`
+	ArtistMBIDs    []mbtypes.MBID `json:"artist_mbids,omitempty"`
-	RecordingMBID mbtypes.MBID   `json:"recording_mbid,omitempty"`
+	Artists        []Artist       `json:"artists,omitempty"`
-	ReleaseMBID   mbtypes.MBID   `json:"release_mbid,omitempty"`
+	RecordingMBID  mbtypes.MBID   `json:"recording_mbid,omitempty"`
-	ArtistMBIDs   []mbtypes.MBID `json:"artist_mbids,omitempty"`
+	RecordingName  string         `json:"recording_name,omitempty"`
-	Artists       []Artist       `json:"artists,omitempty"`
+	ReleaseMBID    mbtypes.MBID   `json:"release_mbid,omitempty"`
+	CAAID          int            `json:"caa_id,omitempty"`
+	CAAReleaseMBID mbtypes.MBID   `json:"caa_release_mbid,omitempty"`
 }
 
 type Artist struct {

internal/listenbrainz/models_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -29,7 +29,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uploadedlobster.com/mbtypes"
-	"go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
+	"go.uploadedlobster.com/scotty/internal/listenbrainz"
 )
 
 func TestTrackDurationMillisecondsInt(t *testing.T) {