diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 0000000..7714552 --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,179 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +// Implements generic access to files inside an archive. +// +// An archive in this context can be any container that holds files. +// In this implementation the archive can be a ZIP file or a directory. +package archive + +import ( + "archive/zip" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" +) + +// Generic archive interface. +type Archive interface { + Close() error + Open(path string) (fs.File, error) + Glob(pattern string) ([]FileInfo, error) +} + +// Open an archive in path. +// The archive can be a ZIP file or a directory. The implementation +// will detect the type of archive and return the appropriate +// implementation of the Archive interface. +func OpenArchive(path string) (Archive, error) { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + switch mode := fi.Mode(); { + case mode.IsRegular(): + archive := &zipArchive{} + err := archive.OpenArchive(path) + if err != nil { + return nil, err + } + return archive, nil + case mode.IsDir(): + archive := &dirArchive{} + err := archive.OpenArchive(path) + if err != nil { + return nil, err + } + return archive, nil + default: + return nil, fmt.Errorf("unsupported file mode: %s", mode) + } +} + +// Interface for a file that can be opened when needed. +type OpenableFile interface { + Open() (io.ReadCloser, error) +} + +// Generic information about a file inside an archive. +type FileInfo struct { + Name string + File OpenableFile +} + +// A openable file in the filesystem. +type filesystemFile struct { + path string +} + +func (f *filesystemFile) Open() (io.ReadCloser, error) { + return os.Open(f.path) +} + +// An implementation of the archiveBackend interface for zip files. +type zipArchive struct { + zip *zip.ReadCloser +} + +func (a *zipArchive) OpenArchive(path string) error { + zip, err := zip.OpenReader(path) + if err != nil { + return err + } + a.zip = zip + return nil +} + +func (a *zipArchive) Close() error { + if a.zip == nil { + return nil + } + return a.zip.Close() +} + +func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { + result := make([]FileInfo, 0) + for _, file := range a.zip.File { + if matched, err := filepath.Match(pattern, file.Name); matched { + if err != nil { + return nil, err + } + info := FileInfo{ + Name: file.Name, + File: file, + } + result = append(result, info) + } + } + + return result, nil +} + +func (a *zipArchive) Open(path string) (fs.File, error) { + file, err := a.zip.Open(path) + if err != nil { + return nil, err + } + return file, nil +} + +// An implementation of the archiveBackend interface for directories. +type dirArchive struct { + path string + dirFS fs.FS +} + +func (a *dirArchive) OpenArchive(path string) error { + a.path = filepath.Clean(path) + a.dirFS = os.DirFS(path) + return nil +} + +func (a *dirArchive) Close() error { + return nil +} + +// Open opens the named file in the archive. +// [fs.File.Close] must be called to release any associated resources. +func (a *dirArchive) Open(path string) (fs.File, error) { + return a.dirFS.Open(path) +} + +func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { + files, err := fs.Glob(a.dirFS, pattern) + if err != nil { + return nil, err + } + result := make([]FileInfo, 0) + for _, name := range files { + fullPath := filepath.Join(a.path, name) + info := FileInfo{ + Name: name, + File: &filesystemFile{path: fullPath}, + } + result = append(result, info) + } + + return result, nil +} diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go index 143a674..0848d38 100644 --- a/internal/backends/lbarchive/lbarchive.go +++ b/internal/backends/lbarchive/lbarchive.go @@ -28,8 +28,8 @@ import ( lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" + "go.uploadedlobster.com/scotty/internal/listenbrainz" "go.uploadedlobster.com/scotty/internal/models" - "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) const batchSize = 2000 @@ -69,7 +69,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens( }, } - archive, err := listenbrainz.OpenArchive(b.filePath) + archive, err := listenbrainz.OpenExportArchive(b.filePath) if err != nil { p.Export.Abort() progress <- p diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 5e80a10..4f0ce2f 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -26,10 +26,10 @@ import ( "go.uploadedlobster.com/musicbrainzws2" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" + "go.uploadedlobster.com/scotty/internal/listenbrainz" "go.uploadedlobster.com/scotty/internal/models" "go.uploadedlobster.com/scotty/internal/similarity" "go.uploadedlobster.com/scotty/internal/version" - "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) type ListenBrainzApiBackend struct { diff --git a/internal/backends/listenbrainz/listenbrainz_test.go b/internal/backends/listenbrainz/listenbrainz_test.go index dd3e1d3..f7151e5 100644 --- a/internal/backends/listenbrainz/listenbrainz_test.go +++ b/internal/backends/listenbrainz/listenbrainz_test.go @@ -26,7 +26,7 @@ import ( "go.uploadedlobster.com/mbtypes" lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" - "go.uploadedlobster.com/scotty/pkg/listenbrainz" + "go.uploadedlobster.com/scotty/internal/listenbrainz" ) func TestInitConfig(t *testing.T) { diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index 76d0c9e..ce470ff 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -20,7 +20,6 @@ package spotifyhistory import ( "context" "os" - "path" "path/filepath" "slices" "sort" @@ -74,7 +73,7 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error { } func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { - files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob)) + files, err := filepath.Glob(filepath.Join(b.dirPath, historyFileGlob)) p := models.TransferProgress{ Export: &models.Progress{}, } diff --git a/internal/config/config.go b/internal/config/config.go index a529b92..94da799 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "os" - "path" "path/filepath" "regexp" "strings" @@ -40,7 +39,7 @@ const ( func DefaultConfigDir() string { configDir, err := os.UserConfigDir() cobra.CheckErr(err) - return path.Join(configDir, version.AppName) + return filepath.Join(configDir, version.AppName) } // initConfig reads in config file and ENV variables if set. diff --git a/pkg/listenbrainz/archive.go b/internal/listenbrainz/archive.go similarity index 67% rename from pkg/listenbrainz/archive.go rename to internal/listenbrainz/archive.go index 668b7e1..b7b5909 100644 --- a/pkg/listenbrainz/archive.go +++ b/internal/listenbrainz/archive.go @@ -22,36 +22,48 @@ THE SOFTWARE. package listenbrainz import ( - "archive/zip" "encoding/json" - "fmt" + "errors" "io" "iter" - "os" "regexp" "sort" "strconv" "time" "github.com/simonfrey/jsonl" + "go.uploadedlobster.com/scotty/internal/archive" ) // Represents a ListenBrainz export archive. // // The export contains the user's listen history, favorite tracks and // user information. -type Archive struct { - backend archiveBackend +type ExportArchive struct { + backend archive.Archive +} + +// Open a ListenBrainz archive from file path. +func OpenExportArchive(path string) (*ExportArchive, error) { + backend, err := archive.OpenArchive(path) + if err != nil { + return nil, err + } + + return &ExportArchive{backend: backend}, nil } // Close the archive and release any resources. -func (a *Archive) Close() error { +func (a *ExportArchive) Close() error { + if a.backend == nil { + return nil + } return a.backend.Close() } // Read the user information from the archive. -func (a *Archive) UserInfo() (UserInfo, error) { - f, err := a.backend.OpenUserInfoFile() +func (a *ExportArchive) UserInfo() (UserInfo, error) { + f, err := a.backend.Open("user.json") if err != nil { return UserInfo{}, err } @@ -67,11 +79,43 @@ func (a *Archive) UserInfo() (UserInfo, error) { return userInfo, nil } +func (a *ExportArchive) ListListenExports() ([]ListenExportFileInfo, error) { + re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`) + result := make([]ListenExportFileInfo, 0) + + files, err := a.backend.Glob("listens/*/*.jsonl") + if err != nil { + return nil, err + } + + for _, file := range files { + match := re.FindStringSubmatch(file.Name) + if match == nil { + continue + } + + year := match[1] + month := match[2] + times, err := getMonthTimeRange(year, month) + if err != nil { + return nil, err + } + info := ListenExportFileInfo{ + Name: file.Name, + TimeRange: *times, + f: file.File, + } + result = append(result, info) + } + + return result, nil +} + // Yields all listens from the archive that are newer than the given timestamp. // The listens are yielded in ascending order of their listened_at timestamp. -func (a *Archive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { +func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { return func(yield func(Listen, error) bool) { - files, err := a.backend.ListListenExports() + files, err := a.ListListenExports() if err != nil { yield(Listen{}, err) return @@ -86,8 +130,8 @@ func (a *Archive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { continue } - f := NewExportFile(file.f) - for l, err := range f.IterListens() { + f := JSONLFile[Listen]{file: file.f} + for l, err := range f.IterItems() { if err != nil { yield(Listen{}, err) return @@ -104,25 +148,33 @@ func (a *Archive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { } } -// Open a ListenBrainz archive from file path. -func OpenArchive(path string) (*Archive, error) { - fi, err := os.Stat(path) - if err != nil { - return nil, err - } - switch mode := fi.Mode(); { - case mode.IsRegular(): - backend := &zipArchive{} - err := backend.Open(path) +// Yields all feedbacks from the archive that are newer than the given timestamp. +// The feedbacks are yielded in ascending order of their Created timestamp. +func (a *ExportArchive) IterFeedback(minTimestamp time.Time) iter.Seq2[Feedback, error] { + return func(yield func(Feedback, error) bool) { + files, err := a.backend.Glob("feedback.jsonl") if err != nil { - return nil, err + yield(Feedback{}, err) + return + } else if len(files) == 0 { + yield(Feedback{}, errors.New("no feedback.jsonl file found in archive")) + return + } + + j := JSONLFile[Feedback]{file: files[0].File} + for l, err := range j.IterItems() { + if err != nil { + yield(Feedback{}, err) + return + } + + if !time.Unix(l.Created, 0).After(minTimestamp) { + continue + } + if !yield(l, nil) { + break + } } - return &Archive{backend: backend}, nil - case mode.IsDir(): - // TODO: Implement directory mode - return nil, fmt.Errorf("directory mode not implemented") - default: - return nil, fmt.Errorf("unsupported file mode: %s", mode) } } @@ -131,91 +183,22 @@ type UserInfo struct { Name string `json:"username"` } -type archiveBackend interface { - Close() error - OpenUserInfoFile() (io.ReadCloser, error) - ListListenExports() ([]ListenExportFileInfo, error) -} - type timeRange struct { Start time.Time End time.Time } -type openableFile interface { - Open() (io.ReadCloser, error) -} - type ListenExportFileInfo struct { Name string TimeRange timeRange - f openableFile + f archive.OpenableFile } -type zipArchive struct { - zip *zip.ReadCloser +type JSONLFile[T any] struct { + file archive.OpenableFile } -func (a *zipArchive) Open(path string) error { - zip, err := zip.OpenReader(path) - if err != nil { - return err - } - a.zip = zip - return nil -} - -func (a *zipArchive) Close() error { - if a.zip == nil { - return nil - } - return a.zip.Close() -} - -func (a *zipArchive) OpenUserInfoFile() (io.ReadCloser, error) { - file, err := a.zip.Open("user.json") - if err != nil { - return nil, err - } - return file, nil -} - -func (a *zipArchive) ListListenExports() ([]ListenExportFileInfo, error) { - re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`) - result := make([]ListenExportFileInfo, 0) - - for _, file := range a.zip.File { - match := re.FindStringSubmatch(file.Name) - if match == nil { - continue - } - - year := match[1] - month := match[2] - times, err := getMonthTimeRange(year, month) - if err != nil { - return nil, err - } - info := ListenExportFileInfo{ - Name: file.Name, - TimeRange: *times, - f: file, - } - result = append(result, info) - } - - return result, nil -} - -type ListenExportFile struct { - file openableFile -} - -func NewExportFile(f openableFile) ListenExportFile { - return ListenExportFile{file: f} -} - -func (f *ListenExportFile) openReader() (*jsonl.Reader, error) { +func (f *JSONLFile[T]) openReader() (*jsonl.Reader, error) { fio, err := f.file.Open() if err != nil { return nil, err @@ -224,17 +207,18 @@ func (f *ListenExportFile) openReader() (*jsonl.Reader, error) { return &reader, nil } -func (f *ListenExportFile) IterListens() iter.Seq2[Listen, error] { - return func(yield func(Listen, error) bool) { +func (f *JSONLFile[T]) IterItems() iter.Seq2[T, error] { + return func(yield func(T, error) bool) { reader, err := f.openReader() if err != nil { - yield(Listen{}, err) + var listen T + yield(listen, err) return } defer reader.Close() for { - listen := Listen{} + var listen T err := reader.ReadSingleLine(&listen) if err != nil { break diff --git a/pkg/listenbrainz/client.go b/internal/listenbrainz/client.go similarity index 100% rename from pkg/listenbrainz/client.go rename to internal/listenbrainz/client.go diff --git a/pkg/listenbrainz/client_test.go b/internal/listenbrainz/client_test.go similarity index 99% rename from pkg/listenbrainz/client_test.go rename to internal/listenbrainz/client_test.go index 3742ca9..9baf293 100644 --- a/pkg/listenbrainz/client_test.go +++ b/internal/listenbrainz/client_test.go @@ -31,7 +31,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/pkg/listenbrainz" + "go.uploadedlobster.com/scotty/internal/listenbrainz" ) func TestNewClient(t *testing.T) { diff --git a/pkg/listenbrainz/models.go b/internal/listenbrainz/models.go similarity index 100% rename from pkg/listenbrainz/models.go rename to internal/listenbrainz/models.go diff --git a/pkg/listenbrainz/models_test.go b/internal/listenbrainz/models_test.go similarity index 98% rename from pkg/listenbrainz/models_test.go rename to internal/listenbrainz/models_test.go index 8fb4994..404b87b 100644 --- a/pkg/listenbrainz/models_test.go +++ b/internal/listenbrainz/models_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/pkg/listenbrainz" + "go.uploadedlobster.com/scotty/internal/listenbrainz" ) func TestTrackDurationMillisecondsInt(t *testing.T) { diff --git a/pkg/listenbrainz/testdata/feedback.json b/internal/listenbrainz/testdata/feedback.json similarity index 100% rename from pkg/listenbrainz/testdata/feedback.json rename to internal/listenbrainz/testdata/feedback.json diff --git a/pkg/listenbrainz/testdata/listen.json b/internal/listenbrainz/testdata/listen.json similarity index 100% rename from pkg/listenbrainz/testdata/listen.json rename to internal/listenbrainz/testdata/listen.json diff --git a/pkg/listenbrainz/testdata/listens.json b/internal/listenbrainz/testdata/listens.json similarity index 100% rename from pkg/listenbrainz/testdata/listens.json rename to internal/listenbrainz/testdata/listens.json diff --git a/pkg/listenbrainz/testdata/lookup.json b/internal/listenbrainz/testdata/lookup.json similarity index 100% rename from pkg/listenbrainz/testdata/lookup.json rename to internal/listenbrainz/testdata/lookup.json