Implemented listenbrainz-archive backend with listen export support

internal/archive/archive.go

@@ -1,179 +0,0 @@
-/*
-Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-// Implements generic access to files inside an archive.
-//
-// An archive in this context can be any container that holds files.
-// In this implementation the archive can be a ZIP file or a directory.
-package archive
-
-import (
-	"archive/zip"
-	"fmt"
-	"io"
-	"io/fs"
-	"os"
-	"path/filepath"
-)
-
-// Generic archive interface.
-type Archive interface {
-	Close() error
-	Open(path string) (fs.File, error)
-	Glob(pattern string) ([]FileInfo, error)
-}
-
-// Open an archive in path.
-// The archive can be a ZIP file or a directory. The implementation
-// will detect the type of archive and return the appropriate
-// implementation of the Archive interface.
-func OpenArchive(path string) (Archive, error) {
-	fi, err := os.Stat(path)
-	if err != nil {
-		return nil, err
-	}
-	switch mode := fi.Mode(); {
-	case mode.IsRegular():
-		archive := &zipArchive{}
-		err := archive.OpenArchive(path)
-		if err != nil {
-			return nil, err
-		}
-		return archive, nil
-	case mode.IsDir():
-		archive := &dirArchive{}
-		err := archive.OpenArchive(path)
-		if err != nil {
-			return nil, err
-		}
-		return archive, nil
-	default:
-		return nil, fmt.Errorf("unsupported file mode: %s", mode)
-	}
-}
-
-// Interface for a file that can be opened when needed.
-type OpenableFile interface {
-	Open() (io.ReadCloser, error)
-}
-
-// Generic information about a file inside an archive.
-type FileInfo struct {
-	Name string
-	File OpenableFile
-}
-
-// A openable file in the filesystem.
-type filesystemFile struct {
-	path string
-}
-
-func (f *filesystemFile) Open() (io.ReadCloser, error) {
-	return os.Open(f.path)
-}
-
-// An implementation of the archiveBackend interface for zip files.
-type zipArchive struct {
-	zip *zip.ReadCloser
-}
-
-func (a *zipArchive) OpenArchive(path string) error {
-	zip, err := zip.OpenReader(path)
-	if err != nil {
-		return err
-	}
-	a.zip = zip
-	return nil
-}
-
-func (a *zipArchive) Close() error {
-	if a.zip == nil {
-		return nil
-	}
-	return a.zip.Close()
-}
-
-func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) {
-	result := make([]FileInfo, 0)
-	for _, file := range a.zip.File {
-		if matched, err := filepath.Match(pattern, file.Name); matched {
-			if err != nil {
-				return nil, err
-			}
-			info := FileInfo{
-				Name: file.Name,
-				File: file,
-			}
-			result = append(result, info)
-		}
-	}
-
-	return result, nil
-}
-
-func (a *zipArchive) Open(path string) (fs.File, error) {
-	file, err := a.zip.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	return file, nil
-}
-
-// An implementation of the archiveBackend interface for directories.
-type dirArchive struct {
-	path  string
-	dirFS fs.FS
-}
-
-func (a *dirArchive) OpenArchive(path string) error {
-	a.path = filepath.Clean(path)
-	a.dirFS = os.DirFS(path)
-	return nil
-}
-
-func (a *dirArchive) Close() error {
-	return nil
-}
-
-// Open opens the named file in the archive.
-// [fs.File.Close] must be called to release any associated resources.
-func (a *dirArchive) Open(path string) (fs.File, error) {
-	return a.dirFS.Open(path)
-}
-
-func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) {
-	files, err := fs.Glob(a.dirFS, pattern)
-	if err != nil {
-		return nil, err
-	}
-	result := make([]FileInfo, 0)
-	for _, name := range files {
-		fullPath := filepath.Join(a.path, name)
-		info := FileInfo{
-			Name: name,
-			File: &filesystemFile{path: fullPath},
-		}
-		result = append(result, info)
-	}
-
-	return result, nil
-}

internal/backends/lbarchive/lbarchive.go

@@ -28,8 +28,8 @@ import (
 	lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
 	"go.uploadedlobster.com/scotty/internal/config"
 	"go.uploadedlobster.com/scotty/internal/i18n"
-	"go.uploadedlobster.com/scotty/internal/listenbrainz"
 	"go.uploadedlobster.com/scotty/internal/models"
+	"go.uploadedlobster.com/scotty/pkg/listenbrainz"
 )
 
 const batchSize = 2000
@@ -69,7 +69,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens(
 		},
 	}
 
-	archive, err := listenbrainz.OpenExportArchive(b.filePath)
+	archive, err := listenbrainz.OpenArchive(b.filePath)
 	if err != nil {
 		p.Export.Abort()
 		progress <- p

internal/backends/listenbrainz/listenbrainz.go

@@ -26,10 +26,10 @@ import (
 	"go.uploadedlobster.com/musicbrainzws2"
 	"go.uploadedlobster.com/scotty/internal/config"
 	"go.uploadedlobster.com/scotty/internal/i18n"
-	"go.uploadedlobster.com/scotty/internal/listenbrainz"
 	"go.uploadedlobster.com/scotty/internal/models"
 	"go.uploadedlobster.com/scotty/internal/similarity"
 	"go.uploadedlobster.com/scotty/internal/version"
+	"go.uploadedlobster.com/scotty/pkg/listenbrainz"
 )
 
 type ListenBrainzApiBackend struct {

internal/backends/listenbrainz/listenbrainz_test.go

@@ -26,7 +26,7 @@ import (
 	"go.uploadedlobster.com/mbtypes"
 	lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
 	"go.uploadedlobster.com/scotty/internal/config"
-	"go.uploadedlobster.com/scotty/internal/listenbrainz"
+	"go.uploadedlobster.com/scotty/pkg/listenbrainz"
 )
 
 func TestInitConfig(t *testing.T) {

internal/backends/spotifyhistory/spotifyhistory.go

@@ -20,6 +20,7 @@ package spotifyhistory
 import (
 	"context"
 	"os"
+	"path"
 	"path/filepath"
 	"slices"
 	"sort"
@@ -73,7 +74,7 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error {
 }
 
 func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
-	files, err := filepath.Glob(filepath.Join(b.dirPath, historyFileGlob))
+	files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob))
 	p := models.TransferProgress{
 		Export: &models.Progress{},
 	}

internal/config/config.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -39,7 +40,7 @@ const (
 func DefaultConfigDir() string {
 	configDir, err := os.UserConfigDir()
 	cobra.CheckErr(err)
-	return filepath.Join(configDir, version.AppName)
+	return path.Join(configDir, version.AppName)
 }
 
 // initConfig reads in config file and ENV variables if set.

pkg/listenbrainz/archive.go

@@ -22,48 +22,36 @@ THE SOFTWARE.
 package listenbrainz
 
 import (
+	"archive/zip"
 	"encoding/json"
-	"errors"
+	"fmt"
 	"io"
 	"iter"
+	"os"
 	"regexp"
 	"sort"
 	"strconv"
 	"time"
 
 	"github.com/simonfrey/jsonl"
-	"go.uploadedlobster.com/scotty/internal/archive"
 )
 
 // Represents a ListenBrainz export archive.
 //
 // The export contains the user's listen history, favorite tracks and
 // user information.
-type ExportArchive struct {
+type Archive struct {
-	backend archive.Archive
+	backend archiveBackend
-}
-
-// Open a ListenBrainz archive from file path.
-func OpenExportArchive(path string) (*ExportArchive, error) {
-	backend, err := archive.OpenArchive(path)
-	if err != nil {
-		return nil, err
-	}
-
-	return &ExportArchive{backend: backend}, nil
 }
 
 // Close the archive and release any resources.
-func (a *ExportArchive) Close() error {
+func (a *Archive) Close() error {
-	if a.backend == nil {
-		return nil
-	}
 	return a.backend.Close()
 }
 
 // Read the user information from the archive.
-func (a *ExportArchive) UserInfo() (UserInfo, error) {
+func (a *Archive) UserInfo() (UserInfo, error) {
-	f, err := a.backend.Open("user.json")
+	f, err := a.backend.OpenUserInfoFile()
 	if err != nil {
 		return UserInfo{}, err
 	}
@@ -79,43 +67,11 @@ func (a *ExportArchive) UserInfo() (UserInfo, error) {
 	return userInfo, nil
 }
 
-func (a *ExportArchive) ListListenExports() ([]ListenExportFileInfo, error) {
-	re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`)
-	result := make([]ListenExportFileInfo, 0)
-
-	files, err := a.backend.Glob("listens/*/*.jsonl")
-	if err != nil {
-		return nil, err
-	}
-
-	for _, file := range files {
-		match := re.FindStringSubmatch(file.Name)
-		if match == nil {
-			continue
-		}
-
-		year := match[1]
-		month := match[2]
-		times, err := getMonthTimeRange(year, month)
-		if err != nil {
-			return nil, err
-		}
-		info := ListenExportFileInfo{
-			Name:      file.Name,
-			TimeRange: *times,
-			f:         file.File,
-		}
-		result = append(result, info)
-	}
-
-	return result, nil
-}
-
 // Yields all listens from the archive that are newer than the given timestamp.
 // The listens are yielded in ascending order of their listened_at timestamp.
-func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] {
+func (a *Archive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] {
 	return func(yield func(Listen, error) bool) {
-		files, err := a.ListListenExports()
+		files, err := a.backend.ListListenExports()
 		if err != nil {
 			yield(Listen{}, err)
 			return
@@ -130,8 +86,8 @@ func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, er
 				continue
 			}
 
-			f := JSONLFile[Listen]{file: file.f}
+			f := NewExportFile(file.f)
-			for l, err := range f.IterItems() {
+			for l, err := range f.IterListens() {
 				if err != nil {
 					yield(Listen{}, err)
 					return
@@ -148,33 +104,25 @@ func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, er
 	}
 }
 
-// Yields all feedbacks from the archive that are newer than the given timestamp.
+// Open a ListenBrainz archive from file path.
-// The feedbacks are yielded in ascending order of their Created timestamp.
+func OpenArchive(path string) (*Archive, error) {
-func (a *ExportArchive) IterFeedback(minTimestamp time.Time) iter.Seq2[Feedback, error] {
+	fi, err := os.Stat(path)
-	return func(yield func(Feedback, error) bool) {
-		files, err := a.backend.Glob("feedback.jsonl")
 	if err != nil {
-			yield(Feedback{}, err)
+		return nil, err
-			return
-		} else if len(files) == 0 {
-			yield(Feedback{}, errors.New("no feedback.jsonl file found in archive"))
-			return
 	}
-
+	switch mode := fi.Mode(); {
-		j := JSONLFile[Feedback]{file: files[0].File}
+	case mode.IsRegular():
-		for l, err := range j.IterItems() {
+		backend := &zipArchive{}
+		err := backend.Open(path)
 		if err != nil {
-				yield(Feedback{}, err)
+			return nil, err
-				return
-			}
-
-			if !time.Unix(l.Created, 0).After(minTimestamp) {
-				continue
-			}
-			if !yield(l, nil) {
-				break
-			}
 		}
+		return &Archive{backend: backend}, nil
+	case mode.IsDir():
+		// TODO: Implement directory mode
+		return nil, fmt.Errorf("directory mode not implemented")
+	default:
+		return nil, fmt.Errorf("unsupported file mode: %s", mode)
 	}
 }
 
@@ -183,22 +131,91 @@ type UserInfo struct {
 	Name string `json:"username"`
 }
 
+type archiveBackend interface {
+	Close() error
+	OpenUserInfoFile() (io.ReadCloser, error)
+	ListListenExports() ([]ListenExportFileInfo, error)
+}
+
 type timeRange struct {
 	Start time.Time
 	End   time.Time
 }
 
+type openableFile interface {
+	Open() (io.ReadCloser, error)
+}
+
 type ListenExportFileInfo struct {
 	Name      string
 	TimeRange timeRange
-	f         archive.OpenableFile
+	f         openableFile
 }
 
-type JSONLFile[T any] struct {
+type zipArchive struct {
-	file archive.OpenableFile
+	zip *zip.ReadCloser
 }
 
-func (f *JSONLFile[T]) openReader() (*jsonl.Reader, error) {
+func (a *zipArchive) Open(path string) error {
+	zip, err := zip.OpenReader(path)
+	if err != nil {
+		return err
+	}
+	a.zip = zip
+	return nil
+}
+
+func (a *zipArchive) Close() error {
+	if a.zip == nil {
+		return nil
+	}
+	return a.zip.Close()
+}
+
+func (a *zipArchive) OpenUserInfoFile() (io.ReadCloser, error) {
+	file, err := a.zip.Open("user.json")
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+func (a *zipArchive) ListListenExports() ([]ListenExportFileInfo, error) {
+	re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`)
+	result := make([]ListenExportFileInfo, 0)
+
+	for _, file := range a.zip.File {
+		match := re.FindStringSubmatch(file.Name)
+		if match == nil {
+			continue
+		}
+
+		year := match[1]
+		month := match[2]
+		times, err := getMonthTimeRange(year, month)
+		if err != nil {
+			return nil, err
+		}
+		info := ListenExportFileInfo{
+			Name:      file.Name,
+			TimeRange: *times,
+			f:         file,
+		}
+		result = append(result, info)
+	}
+
+	return result, nil
+}
+
+type ListenExportFile struct {
+	file openableFile
+}
+
+func NewExportFile(f openableFile) ListenExportFile {
+	return ListenExportFile{file: f}
+}
+
+func (f *ListenExportFile) openReader() (*jsonl.Reader, error) {
 	fio, err := f.file.Open()
 	if err != nil {
 		return nil, err
@@ -207,18 +224,17 @@ func (f *JSONLFile[T]) openReader() (*jsonl.Reader, error) {
 	return &reader, nil
 }
 
-func (f *JSONLFile[T]) IterItems() iter.Seq2[T, error] {
+func (f *ListenExportFile) IterListens() iter.Seq2[Listen, error] {
-	return func(yield func(T, error) bool) {
+	return func(yield func(Listen, error) bool) {
 		reader, err := f.openReader()
 		if err != nil {
-			var listen T
+			yield(Listen{}, err)
-			yield(listen, err)
 			return
 		}
 		defer reader.Close()
 
 		for {
-			var listen T
+			listen := Listen{}
 			err := reader.ReadSingleLine(&listen)
 			if err != nil {
 				break

pkg/listenbrainz/client_test.go

@@ -31,7 +31,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uploadedlobster.com/mbtypes"
-	"go.uploadedlobster.com/scotty/internal/listenbrainz"
+	"go.uploadedlobster.com/scotty/pkg/listenbrainz"
 )
 
 func TestNewClient(t *testing.T) {

pkg/listenbrainz/models_test.go

@@ -29,7 +29,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uploadedlobster.com/mbtypes"
-	"go.uploadedlobster.com/scotty/internal/listenbrainz"
+	"go.uploadedlobster.com/scotty/pkg/listenbrainz"
 )
 
 func TestTrackDurationMillisecondsInt(t *testing.T) {