Implemented listenbrainz-archive backend with listen export support

README.md

@@ -117,18 +117,19 @@ scotty beam listens deezer listenbrainz --timestamp "2023-12-06 14:26:24"
 ### Supported backends
 The following table lists the available backends and the currently supported features.
 
-Backend         | Listens Export | Listens Import | Loves Export | Loves Import
+Backend              | Listens Export | Listens Import | Loves Export | Loves Import
-----------------|----------------|----------------|--------------|-------------
+---------------------|----------------|----------------|--------------|-------------
-deezer          | ✓              | ⨯              | ✓            | -
+deezer               | ✓              | ⨯              | ✓            | -
-funkwhale       | ✓              | ⨯              | ✓            | -
+funkwhale            | ✓              | ⨯              | ✓            | -
-jspf            | ✓              | ✓              | ✓            | ✓
+jspf                 | ✓              | ✓              | ✓            | ✓
-lastfm          | ✓              | ✓              | ✓            | ✓
+lastfm               | ✓              | ✓              | ✓            | ✓
-listenbrainz    | ✓              | ✓              | ✓            | ✓
+listenbrainz         | ✓              | ✓              | ✓            | ✓
-maloja          | ✓              | ✓              | ⨯            | ⨯
+listenbrainz-archive | ✓              | -              | -            | -
-scrobbler-log   | ✓              | ✓              | ⨯            | ⨯
+maloja               | ✓              | ✓              | ⨯            | ⨯
-spotify         | ✓              | ⨯              | ✓            | -
+scrobbler-log        | ✓              | ✓              | ⨯            | ⨯
-spotify-history | ✓              | ⨯              | ⨯            | ⨯
+spotify              | ✓              | ⨯              | ✓            | -
-subsonic        | ⨯              | ⨯              | ✓            | -
+spotify-history      | ✓              | ⨯              | ⨯            | ⨯
+subsonic             | ⨯              | ⨯              | ✓            | -
 
 **✓** implemented **-** not yet implemented **⨯** unavailable / not planned
 

config.example.toml

@@ -19,6 +19,13 @@ token    = ""
 # not already exists in your ListenBrainz profile.
 check-duplicate-listens = false
 
+[service.listenbrainz-archive]
+# This backend supports listens from a ListenBrainz export archive
+# (https://listenbrainz.org/settings/export/).
+backend   = "listenbrainz-archive"
+# The file path to the ListenBrainz export archive.
+file-path = "./listenbrainz_outsidecontext.zip"
+
 [service.maloja]
 # Maloja is a self hosted listening service (https://github.com/krateng/maloja)
 backend    = "maloja"

go.mod

@@ -53,6 +53,7 @@ require (
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.9.0 // indirect
+	github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.14.0 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect

go.sum

@@ -107,6 +107,8 @@ github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFT
 github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk=
 github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0 h1:cgqwZtnR+IQfUYDLJ3Kiy4aE+O/wExTzEIg8xwC4Qfs=
 github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0/go.mod h1:n3nudMl178cEvD44PaopxH9jhJaQzthSxUzLO5iKMy4=
+github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 h1:CXJI+lliMiiEwzfgE8yt/38K0heYDgQ0L3f/3fxRnQU=
+github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740/go.mod h1:G4w16caPmc6at7u4fmkj/8OAoOnM9mkmJr2fvL0vhaw=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
 github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
 github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA=

internal/backends/backends.go

@@ -27,6 +27,7 @@ import (
 	"go.uploadedlobster.com/scotty/internal/backends/funkwhale"
 	"go.uploadedlobster.com/scotty/internal/backends/jspf"
 	"go.uploadedlobster.com/scotty/internal/backends/lastfm"
+	"go.uploadedlobster.com/scotty/internal/backends/lbarchive"
 	"go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
 	"go.uploadedlobster.com/scotty/internal/backends/maloja"
 	"go.uploadedlobster.com/scotty/internal/backends/scrobblerlog"
@@ -105,17 +106,18 @@ func GetBackends() BackendList {
 }
 
 var knownBackends = map[string]func() models.Backend{
-	"deezer":          func() models.Backend { return &deezer.DeezerApiBackend{} },
+	"deezer":               func() models.Backend { return &deezer.DeezerApiBackend{} },
-	"dump":            func() models.Backend { return &dump.DumpBackend{} },
+	"dump":                 func() models.Backend { return &dump.DumpBackend{} },
-	"funkwhale":       func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} },
+	"funkwhale":            func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} },
-	"jspf":            func() models.Backend { return &jspf.JSPFBackend{} },
+	"jspf":                 func() models.Backend { return &jspf.JSPFBackend{} },
-	"lastfm":          func() models.Backend { return &lastfm.LastfmApiBackend{} },
+	"lastfm":               func() models.Backend { return &lastfm.LastfmApiBackend{} },
-	"listenbrainz":    func() models.Backend { return &listenbrainz.ListenBrainzApiBackend{} },
+	"listenbrainz":         func() models.Backend { return &listenbrainz.ListenBrainzApiBackend{} },
-	"maloja":          func() models.Backend { return &maloja.MalojaApiBackend{} },
+	"listenbrainz-archive": func() models.Backend { return &lbarchive.ListenBrainzArchiveBackend{} },
-	"scrobbler-log":   func() models.Backend { return &scrobblerlog.ScrobblerLogBackend{} },
+	"maloja":               func() models.Backend { return &maloja.MalojaApiBackend{} },
-	"spotify":         func() models.Backend { return &spotify.SpotifyApiBackend{} },
+	"scrobbler-log":        func() models.Backend { return &scrobblerlog.ScrobblerLogBackend{} },
-	"spotify-history": func() models.Backend { return &spotifyhistory.SpotifyHistoryBackend{} },
+	"spotify":              func() models.Backend { return &spotify.SpotifyApiBackend{} },
-	"subsonic":        func() models.Backend { return &subsonic.SubsonicApiBackend{} },
+	"spotify-history":      func() models.Backend { return &spotifyhistory.SpotifyHistoryBackend{} },
+	"subsonic":             func() models.Backend { return &subsonic.SubsonicApiBackend{} },
 }
 
 func backendWithConfig(config config.ServiceConfig) (models.Backend, error) {

internal/backends/backends_test.go

@@ -28,6 +28,7 @@ import (
 	"go.uploadedlobster.com/scotty/internal/backends/funkwhale"
 	"go.uploadedlobster.com/scotty/internal/backends/jspf"
 	"go.uploadedlobster.com/scotty/internal/backends/lastfm"
+	"go.uploadedlobster.com/scotty/internal/backends/lbarchive"
 	"go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
 	"go.uploadedlobster.com/scotty/internal/backends/maloja"
 	"go.uploadedlobster.com/scotty/internal/backends/scrobblerlog"
@@ -103,6 +104,11 @@ func TestImplementsInterfaces(t *testing.T) {
 	expectInterface[models.LovesExport](t, &lastfm.LastfmApiBackend{})
 	expectInterface[models.LovesImport](t, &lastfm.LastfmApiBackend{})
 
+	expectInterface[models.ListensExport](t, &lbarchive.ListenBrainzArchiveBackend{})
+	// expectInterface[models.ListensImport](t, &lbarchive.ListenBrainzArchiveBackend{})
+	// expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{})
+	// expectInterface[models.LovesImport](t, &lbarchive.ListenBrainzArchiveBackend{})
+
 	expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{})
 	expectInterface[models.ListensImport](t, &listenbrainz.ListenBrainzApiBackend{})
 	expectInterface[models.LovesExport](t, &listenbrainz.ListenBrainzApiBackend{})

internal/backends/lbarchive/lbarchive.go

@@ -0,0 +1,121 @@
+/*
+Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package lbarchive
+
+import (
+	"context"
+	"time"
+
+	lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
+	"go.uploadedlobster.com/scotty/internal/config"
+	"go.uploadedlobster.com/scotty/internal/i18n"
+	"go.uploadedlobster.com/scotty/internal/models"
+	"go.uploadedlobster.com/scotty/pkg/listenbrainz"
+)
+
+const batchSize = 2000
+
+type ListenBrainzArchiveBackend struct {
+	filePath string
+}
+
+func (b *ListenBrainzArchiveBackend) Name() string { return "listenbrainz-archive" }
+
+func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption {
+	return []models.BackendOption{{
+		Name:  "file-path",
+		Label: i18n.Tr("Export ZIP file path"),
+		Type:  models.String,
+	}}
+}
+
+func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error {
+	b.filePath = config.GetString("file-path")
+	return nil
+}
+
+func (b *ListenBrainzArchiveBackend) ExportListens(
+	ctx context.Context, oldestTimestamp time.Time,
+	results chan models.ListensResult, progress chan models.TransferProgress) {
+	startTime := time.Now()
+	minTime := oldestTimestamp
+	if minTime.Unix() < 1 {
+		minTime = time.Unix(1, 0)
+	}
+
+	totalDuration := startTime.Sub(oldestTimestamp)
+	p := models.TransferProgress{
+		Export: &models.Progress{
+			Total: int64(totalDuration.Seconds()),
+		},
+	}
+
+	archive, err := listenbrainz.OpenArchive(b.filePath)
+	if err != nil {
+		p.Export.Abort()
+		progress <- p
+		results <- models.ListensResult{Error: err}
+		return
+	}
+	defer archive.Close()
+
+	userInfo, err := archive.UserInfo()
+	if err != nil {
+		p.Export.Abort()
+		progress <- p
+		results <- models.ListensResult{Error: err}
+		return
+	}
+
+	listens := make(models.ListensList, 0, batchSize)
+	for rawListen, err := range archive.IterListens(oldestTimestamp) {
+		if err != nil {
+			p.Export.Abort()
+			progress <- p
+			results <- models.ListensResult{Error: err}
+			return
+		}
+
+		listen := lbapi.AsListen(rawListen)
+		if listen.UserName == "" {
+			listen.UserName = userInfo.Name
+		}
+		listens = append(listens, listen)
+
+		// Update the progress
+		p.Export.TotalItems += 1
+		remainingTime := startTime.Sub(listen.ListenedAt)
+		p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
+
+		// Allow the importer to start processing the listens by
+		// sending them in batches.
+		if len(listens) >= batchSize {
+			results <- models.ListensResult{Items: listens}
+			progress <- p
+			listens = listens[:0]
+		}
+	}
+
+	results <- models.ListensResult{Items: listens}
+	p.Export.Complete()
+	progress <- p
+}

internal/backends/lbarchive/lbarchive_test.go

@@ -0,0 +1,40 @@
+/*
+Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package lbarchive_test
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"go.uploadedlobster.com/scotty/internal/backends/lbarchive"
+	"go.uploadedlobster.com/scotty/internal/config"
+)
+
+func TestInitConfig(t *testing.T) {
+	c := viper.New()
+	c.Set("file-path", "/foo/lbarchive.zip")
+	service := config.NewServiceConfig("test", c)
+	backend := lbarchive.ListenBrainzArchiveBackend{}
+	err := backend.InitConfig(&service)
+	assert.NoError(t, err)
+}

pkg/listenbrainz/archive.go

@@ -0,0 +1,267 @@
+/*
+Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package listenbrainz
+
+import (
+	"archive/zip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"iter"
+	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/simonfrey/jsonl"
+)
+
+// Represents a ListenBrainz export archive.
+//
+// The export contains the user's listen history, favorite tracks and
+// user information.
+type Archive struct {
+	backend archiveBackend
+}
+
+// Close the archive and release any resources.
+func (a *Archive) Close() error {
+	return a.backend.Close()
+}
+
+// Read the user information from the archive.
+func (a *Archive) UserInfo() (UserInfo, error) {
+	f, err := a.backend.OpenUserInfoFile()
+	if err != nil {
+		return UserInfo{}, err
+	}
+	defer f.Close()
+
+	userInfo := UserInfo{}
+	bytes, err := io.ReadAll(f)
+	if err != nil {
+		return userInfo, err
+	}
+
+	json.Unmarshal(bytes, &userInfo)
+	return userInfo, nil
+}
+
+// Yields all listens from the archive that are newer than the given timestamp.
+// The listens are yielded in ascending order of their listened_at timestamp.
+func (a *Archive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] {
+	return func(yield func(Listen, error) bool) {
+		files, err := a.backend.ListListenExports()
+		if err != nil {
+			yield(Listen{}, err)
+			return
+		}
+
+		sort.Slice(files, func(i, j int) bool {
+			return files[i].TimeRange.Start.Before(files[j].TimeRange.Start)
+		})
+
+		for _, file := range files {
+			if file.TimeRange.End.Before(minTimestamp) {
+				continue
+			}
+
+			f := NewExportFile(file.f)
+			for l, err := range f.IterListens() {
+				if err != nil {
+					yield(Listen{}, err)
+					return
+				}
+
+				if !time.Unix(l.ListenedAt, 0).After(minTimestamp) {
+					continue
+				}
+				if !yield(l, nil) {
+					break
+				}
+			}
+		}
+	}
+}
+
+// Open a ListenBrainz archive from file path.
+func OpenArchive(path string) (*Archive, error) {
+	fi, err := os.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+	switch mode := fi.Mode(); {
+	case mode.IsRegular():
+		backend := &zipArchive{}
+		err := backend.Open(path)
+		if err != nil {
+			return nil, err
+		}
+		return &Archive{backend: backend}, nil
+	case mode.IsDir():
+		// TODO: Implement directory mode
+		return nil, fmt.Errorf("directory mode not implemented")
+	default:
+		return nil, fmt.Errorf("unsupported file mode: %s", mode)
+	}
+}
+
+type UserInfo struct {
+	ID   string `json:"user_id"`
+	Name string `json:"username"`
+}
+
+type archiveBackend interface {
+	Close() error
+	OpenUserInfoFile() (io.ReadCloser, error)
+	ListListenExports() ([]ListenExportFileInfo, error)
+}
+
+type timeRange struct {
+	Start time.Time
+	End   time.Time
+}
+
+type openableFile interface {
+	Open() (io.ReadCloser, error)
+}
+
+type ListenExportFileInfo struct {
+	Name      string
+	TimeRange timeRange
+	f         openableFile
+}
+
+type zipArchive struct {
+	zip *zip.ReadCloser
+}
+
+func (a *zipArchive) Open(path string) error {
+	zip, err := zip.OpenReader(path)
+	if err != nil {
+		return err
+	}
+	a.zip = zip
+	return nil
+}
+
+func (a *zipArchive) Close() error {
+	if a.zip == nil {
+		return nil
+	}
+	return a.zip.Close()
+}
+
+func (a *zipArchive) OpenUserInfoFile() (io.ReadCloser, error) {
+	file, err := a.zip.Open("user.json")
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+func (a *zipArchive) ListListenExports() ([]ListenExportFileInfo, error) {
+	re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`)
+	result := make([]ListenExportFileInfo, 0)
+
+	for _, file := range a.zip.File {
+		match := re.FindStringSubmatch(file.Name)
+		if match == nil {
+			continue
+		}
+
+		year := match[1]
+		month := match[2]
+		times, err := getMonthTimeRange(year, month)
+		if err != nil {
+			return nil, err
+		}
+		info := ListenExportFileInfo{
+			Name:      file.Name,
+			TimeRange: *times,
+			f:         file,
+		}
+		result = append(result, info)
+	}
+
+	return result, nil
+}
+
+type ListenExportFile struct {
+	file openableFile
+}
+
+func NewExportFile(f openableFile) ListenExportFile {
+	return ListenExportFile{file: f}
+}
+
+func (f *ListenExportFile) openReader() (*jsonl.Reader, error) {
+	fio, err := f.file.Open()
+	if err != nil {
+		return nil, err
+	}
+	reader := jsonl.NewReader(fio)
+	return &reader, nil
+}
+
+func (f *ListenExportFile) IterListens() iter.Seq2[Listen, error] {
+	return func(yield func(Listen, error) bool) {
+		reader, err := f.openReader()
+		if err != nil {
+			yield(Listen{}, err)
+			return
+		}
+		defer reader.Close()
+
+		for {
+			listen := Listen{}
+			err := reader.ReadSingleLine(&listen)
+			if err != nil {
+				break
+			}
+			if !yield(listen, nil) {
+				break
+			}
+		}
+	}
+}
+
+func getMonthTimeRange(year string, month string) (*timeRange, error) {
+	yearInt, err := strconv.Atoi(year)
+	if err != nil {
+		return nil, err
+	}
+	monthInt, err := strconv.Atoi(month)
+	if err != nil {
+		return nil, err
+	}
+
+	r := &timeRange{}
+	r.Start = time.Date(yearInt, time.Month(monthInt), 1, 0, 0, 0, 0, time.UTC)
+
+	// Get the end of the month
+	nextMonth := monthInt + 1
+	r.End = time.Date(
+		yearInt, time.Month(nextMonth), 1, 0, 0, 0, 0, time.UTC).Add(-time.Second)
+	return r, nil
+}

pkg/listenbrainz/models.go

@@ -55,11 +55,11 @@ type ListenSubmission struct {
 }
 
 type Listen struct {
-	InsertedAt    int64  `json:"inserted_at,omitempty"`
+	InsertedAt    float64 `json:"inserted_at,omitempty"`
-	ListenedAt    int64  `json:"listened_at"`
+	ListenedAt    int64   `json:"listened_at"`
-	RecordingMSID string `json:"recording_msid,omitempty"`
+	RecordingMSID string  `json:"recording_msid,omitempty"`
-	UserName      string `json:"user_name,omitempty"`
+	UserName      string  `json:"user_name,omitempty"`
-	TrackMetadata Track  `json:"track_metadata"`
+	TrackMetadata Track   `json:"track_metadata"`
 }
 
 type Track struct {