diff --git a/internal/archive/archive.go b/internal/archive/archive.go deleted file mode 100644 index 7714552..0000000 --- a/internal/archive/archive.go +++ /dev/null @@ -1,179 +0,0 @@ -/* -Copyright © 2025 Philipp Wolfer - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -// Implements generic access to files inside an archive. -// -// An archive in this context can be any container that holds files. -// In this implementation the archive can be a ZIP file or a directory. -package archive - -import ( - "archive/zip" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" -) - -// Generic archive interface. -type Archive interface { - Close() error - Open(path string) (fs.File, error) - Glob(pattern string) ([]FileInfo, error) -} - -// Open an archive in path. -// The archive can be a ZIP file or a directory. The implementation -// will detect the type of archive and return the appropriate -// implementation of the Archive interface. -func OpenArchive(path string) (Archive, error) { - fi, err := os.Stat(path) - if err != nil { - return nil, err - } - switch mode := fi.Mode(); { - case mode.IsRegular(): - archive := &zipArchive{} - err := archive.OpenArchive(path) - if err != nil { - return nil, err - } - return archive, nil - case mode.IsDir(): - archive := &dirArchive{} - err := archive.OpenArchive(path) - if err != nil { - return nil, err - } - return archive, nil - default: - return nil, fmt.Errorf("unsupported file mode: %s", mode) - } -} - -// Interface for a file that can be opened when needed. -type OpenableFile interface { - Open() (io.ReadCloser, error) -} - -// Generic information about a file inside an archive. -type FileInfo struct { - Name string - File OpenableFile -} - -// A openable file in the filesystem. -type filesystemFile struct { - path string -} - -func (f *filesystemFile) Open() (io.ReadCloser, error) { - return os.Open(f.path) -} - -// An implementation of the archiveBackend interface for zip files. -type zipArchive struct { - zip *zip.ReadCloser -} - -func (a *zipArchive) OpenArchive(path string) error { - zip, err := zip.OpenReader(path) - if err != nil { - return err - } - a.zip = zip - return nil -} - -func (a *zipArchive) Close() error { - if a.zip == nil { - return nil - } - return a.zip.Close() -} - -func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { - result := make([]FileInfo, 0) - for _, file := range a.zip.File { - if matched, err := filepath.Match(pattern, file.Name); matched { - if err != nil { - return nil, err - } - info := FileInfo{ - Name: file.Name, - File: file, - } - result = append(result, info) - } - } - - return result, nil -} - -func (a *zipArchive) Open(path string) (fs.File, error) { - file, err := a.zip.Open(path) - if err != nil { - return nil, err - } - return file, nil -} - -// An implementation of the archiveBackend interface for directories. -type dirArchive struct { - path string - dirFS fs.FS -} - -func (a *dirArchive) OpenArchive(path string) error { - a.path = filepath.Clean(path) - a.dirFS = os.DirFS(path) - return nil -} - -func (a *dirArchive) Close() error { - return nil -} - -// Open opens the named file in the archive. -// [fs.File.Close] must be called to release any associated resources. -func (a *dirArchive) Open(path string) (fs.File, error) { - return a.dirFS.Open(path) -} - -func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { - files, err := fs.Glob(a.dirFS, pattern) - if err != nil { - return nil, err - } - result := make([]FileInfo, 0) - for _, name := range files { - fullPath := filepath.Join(a.path, name) - info := FileInfo{ - Name: name, - File: &filesystemFile{path: fullPath}, - } - result = append(result, info) - } - - return result, nil -} diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go index 0848d38..143a674 100644 --- a/internal/backends/lbarchive/lbarchive.go +++ b/internal/backends/lbarchive/lbarchive.go @@ -28,8 +28,8 @@ import ( lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" - "go.uploadedlobster.com/scotty/internal/listenbrainz" "go.uploadedlobster.com/scotty/internal/models" + "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) const batchSize = 2000 @@ -69,7 +69,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens( }, } - archive, err := listenbrainz.OpenExportArchive(b.filePath) + archive, err := listenbrainz.OpenArchive(b.filePath) if err != nil { p.Export.Abort() progress <- p diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 4f0ce2f..5e80a10 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -26,10 +26,10 @@ import ( "go.uploadedlobster.com/musicbrainzws2" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" - "go.uploadedlobster.com/scotty/internal/listenbrainz" "go.uploadedlobster.com/scotty/internal/models" "go.uploadedlobster.com/scotty/internal/similarity" "go.uploadedlobster.com/scotty/internal/version" + "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) type ListenBrainzApiBackend struct { diff --git a/internal/backends/listenbrainz/listenbrainz_test.go b/internal/backends/listenbrainz/listenbrainz_test.go index f7151e5..dd3e1d3 100644 --- a/internal/backends/listenbrainz/listenbrainz_test.go +++ b/internal/backends/listenbrainz/listenbrainz_test.go @@ -26,7 +26,7 @@ import ( "go.uploadedlobster.com/mbtypes" lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" - "go.uploadedlobster.com/scotty/internal/listenbrainz" + "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) func TestInitConfig(t *testing.T) { diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index ce470ff..76d0c9e 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -20,6 +20,7 @@ package spotifyhistory import ( "context" "os" + "path" "path/filepath" "slices" "sort" @@ -73,7 +74,7 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error { } func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { - files, err := filepath.Glob(filepath.Join(b.dirPath, historyFileGlob)) + files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob)) p := models.TransferProgress{ Export: &models.Progress{}, } diff --git a/internal/config/config.go b/internal/config/config.go index 94da799..a529b92 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "os" + "path" "path/filepath" "regexp" "strings" @@ -39,7 +40,7 @@ const ( func DefaultConfigDir() string { configDir, err := os.UserConfigDir() cobra.CheckErr(err) - return filepath.Join(configDir, version.AppName) + return path.Join(configDir, version.AppName) } // initConfig reads in config file and ENV variables if set. diff --git a/internal/listenbrainz/archive.go b/pkg/listenbrainz/archive.go similarity index 67% rename from internal/listenbrainz/archive.go rename to pkg/listenbrainz/archive.go index b7b5909..668b7e1 100644 --- a/internal/listenbrainz/archive.go +++ b/pkg/listenbrainz/archive.go @@ -22,48 +22,36 @@ THE SOFTWARE. package listenbrainz import ( + "archive/zip" "encoding/json" - "errors" + "fmt" "io" "iter" + "os" "regexp" "sort" "strconv" "time" "github.com/simonfrey/jsonl" - "go.uploadedlobster.com/scotty/internal/archive" ) // Represents a ListenBrainz export archive. // // The export contains the user's listen history, favorite tracks and // user information. -type ExportArchive struct { - backend archive.Archive -} - -// Open a ListenBrainz archive from file path. -func OpenExportArchive(path string) (*ExportArchive, error) { - backend, err := archive.OpenArchive(path) - if err != nil { - return nil, err - } - - return &ExportArchive{backend: backend}, nil +type Archive struct { + backend archiveBackend } // Close the archive and release any resources. -func (a *ExportArchive) Close() error { - if a.backend == nil { - return nil - } +func (a *Archive) Close() error { return a.backend.Close() } // Read the user information from the archive. -func (a *ExportArchive) UserInfo() (UserInfo, error) { - f, err := a.backend.Open("user.json") +func (a *Archive) UserInfo() (UserInfo, error) { + f, err := a.backend.OpenUserInfoFile() if err != nil { return UserInfo{}, err } @@ -79,43 +67,11 @@ func (a *ExportArchive) UserInfo() (UserInfo, error) { return userInfo, nil } -func (a *ExportArchive) ListListenExports() ([]ListenExportFileInfo, error) { - re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`) - result := make([]ListenExportFileInfo, 0) - - files, err := a.backend.Glob("listens/*/*.jsonl") - if err != nil { - return nil, err - } - - for _, file := range files { - match := re.FindStringSubmatch(file.Name) - if match == nil { - continue - } - - year := match[1] - month := match[2] - times, err := getMonthTimeRange(year, month) - if err != nil { - return nil, err - } - info := ListenExportFileInfo{ - Name: file.Name, - TimeRange: *times, - f: file.File, - } - result = append(result, info) - } - - return result, nil -} - // Yields all listens from the archive that are newer than the given timestamp. // The listens are yielded in ascending order of their listened_at timestamp. -func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { +func (a *Archive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { return func(yield func(Listen, error) bool) { - files, err := a.ListListenExports() + files, err := a.backend.ListListenExports() if err != nil { yield(Listen{}, err) return @@ -130,8 +86,8 @@ func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, er continue } - f := JSONLFile[Listen]{file: file.f} - for l, err := range f.IterItems() { + f := NewExportFile(file.f) + for l, err := range f.IterListens() { if err != nil { yield(Listen{}, err) return @@ -148,33 +104,25 @@ func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, er } } -// Yields all feedbacks from the archive that are newer than the given timestamp. -// The feedbacks are yielded in ascending order of their Created timestamp. -func (a *ExportArchive) IterFeedback(minTimestamp time.Time) iter.Seq2[Feedback, error] { - return func(yield func(Feedback, error) bool) { - files, err := a.backend.Glob("feedback.jsonl") +// Open a ListenBrainz archive from file path. +func OpenArchive(path string) (*Archive, error) { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + switch mode := fi.Mode(); { + case mode.IsRegular(): + backend := &zipArchive{} + err := backend.Open(path) if err != nil { - yield(Feedback{}, err) - return - } else if len(files) == 0 { - yield(Feedback{}, errors.New("no feedback.jsonl file found in archive")) - return - } - - j := JSONLFile[Feedback]{file: files[0].File} - for l, err := range j.IterItems() { - if err != nil { - yield(Feedback{}, err) - return - } - - if !time.Unix(l.Created, 0).After(minTimestamp) { - continue - } - if !yield(l, nil) { - break - } + return nil, err } + return &Archive{backend: backend}, nil + case mode.IsDir(): + // TODO: Implement directory mode + return nil, fmt.Errorf("directory mode not implemented") + default: + return nil, fmt.Errorf("unsupported file mode: %s", mode) } } @@ -183,22 +131,91 @@ type UserInfo struct { Name string `json:"username"` } +type archiveBackend interface { + Close() error + OpenUserInfoFile() (io.ReadCloser, error) + ListListenExports() ([]ListenExportFileInfo, error) +} + type timeRange struct { Start time.Time End time.Time } +type openableFile interface { + Open() (io.ReadCloser, error) +} + type ListenExportFileInfo struct { Name string TimeRange timeRange - f archive.OpenableFile + f openableFile } -type JSONLFile[T any] struct { - file archive.OpenableFile +type zipArchive struct { + zip *zip.ReadCloser } -func (f *JSONLFile[T]) openReader() (*jsonl.Reader, error) { +func (a *zipArchive) Open(path string) error { + zip, err := zip.OpenReader(path) + if err != nil { + return err + } + a.zip = zip + return nil +} + +func (a *zipArchive) Close() error { + if a.zip == nil { + return nil + } + return a.zip.Close() +} + +func (a *zipArchive) OpenUserInfoFile() (io.ReadCloser, error) { + file, err := a.zip.Open("user.json") + if err != nil { + return nil, err + } + return file, nil +} + +func (a *zipArchive) ListListenExports() ([]ListenExportFileInfo, error) { + re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`) + result := make([]ListenExportFileInfo, 0) + + for _, file := range a.zip.File { + match := re.FindStringSubmatch(file.Name) + if match == nil { + continue + } + + year := match[1] + month := match[2] + times, err := getMonthTimeRange(year, month) + if err != nil { + return nil, err + } + info := ListenExportFileInfo{ + Name: file.Name, + TimeRange: *times, + f: file, + } + result = append(result, info) + } + + return result, nil +} + +type ListenExportFile struct { + file openableFile +} + +func NewExportFile(f openableFile) ListenExportFile { + return ListenExportFile{file: f} +} + +func (f *ListenExportFile) openReader() (*jsonl.Reader, error) { fio, err := f.file.Open() if err != nil { return nil, err @@ -207,18 +224,17 @@ func (f *JSONLFile[T]) openReader() (*jsonl.Reader, error) { return &reader, nil } -func (f *JSONLFile[T]) IterItems() iter.Seq2[T, error] { - return func(yield func(T, error) bool) { +func (f *ListenExportFile) IterListens() iter.Seq2[Listen, error] { + return func(yield func(Listen, error) bool) { reader, err := f.openReader() if err != nil { - var listen T - yield(listen, err) + yield(Listen{}, err) return } defer reader.Close() for { - var listen T + listen := Listen{} err := reader.ReadSingleLine(&listen) if err != nil { break diff --git a/internal/listenbrainz/client.go b/pkg/listenbrainz/client.go similarity index 100% rename from internal/listenbrainz/client.go rename to pkg/listenbrainz/client.go diff --git a/internal/listenbrainz/client_test.go b/pkg/listenbrainz/client_test.go similarity index 99% rename from internal/listenbrainz/client_test.go rename to pkg/listenbrainz/client_test.go index 9baf293..3742ca9 100644 --- a/internal/listenbrainz/client_test.go +++ b/pkg/listenbrainz/client_test.go @@ -31,7 +31,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/internal/listenbrainz" + "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) func TestNewClient(t *testing.T) { diff --git a/internal/listenbrainz/models.go b/pkg/listenbrainz/models.go similarity index 100% rename from internal/listenbrainz/models.go rename to pkg/listenbrainz/models.go diff --git a/internal/listenbrainz/models_test.go b/pkg/listenbrainz/models_test.go similarity index 98% rename from internal/listenbrainz/models_test.go rename to pkg/listenbrainz/models_test.go index 404b87b..8fb4994 100644 --- a/internal/listenbrainz/models_test.go +++ b/pkg/listenbrainz/models_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/internal/listenbrainz" + "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) func TestTrackDurationMillisecondsInt(t *testing.T) { diff --git a/internal/listenbrainz/testdata/feedback.json b/pkg/listenbrainz/testdata/feedback.json similarity index 100% rename from internal/listenbrainz/testdata/feedback.json rename to pkg/listenbrainz/testdata/feedback.json diff --git a/internal/listenbrainz/testdata/listen.json b/pkg/listenbrainz/testdata/listen.json similarity index 100% rename from internal/listenbrainz/testdata/listen.json rename to pkg/listenbrainz/testdata/listen.json diff --git a/internal/listenbrainz/testdata/listens.json b/pkg/listenbrainz/testdata/listens.json similarity index 100% rename from internal/listenbrainz/testdata/listens.json rename to pkg/listenbrainz/testdata/listens.json diff --git a/internal/listenbrainz/testdata/lookup.json b/pkg/listenbrainz/testdata/lookup.json similarity index 100% rename from internal/listenbrainz/testdata/lookup.json rename to pkg/listenbrainz/testdata/lookup.json