diff --git a/CHANGES.md b/CHANGES.md index 2257aaa..228b101 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,7 @@ - spotify-history: the parameter to the export archive path has been renamed to `archive-path`. For backward compatibility the old `dir-path` parameter is still read. +- deezer-history: new backend to import listens and loves from Deezer data export. - deezer: fixed endless export loop if the user's listen history was empty. - dump: it is now possible to specify a file to write the text output to. - Fixed potential issues with MusicBrainz rate limiting. diff --git a/README.md b/README.md index b10a030..3c004c0 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,7 @@ The following table lists the available backends and the currently supported fea Backend | Listens Export | Listens Import | Loves Export | Loves Import ---------------------|----------------|----------------|--------------|------------- deezer | ✓ | ⨯ | ✓ | - +deezer-history | ✓ | ⨯ | ✓ | ⨯ funkwhale | ✓ | ⨯ | ✓ | - jspf | ✓ | ✓ | ✓ | ✓ lastfm | ✓ | ✓ | ✓ | ✓ diff --git a/config.example.toml b/config.example.toml index 3acdf88..91d5318 100644 --- a/config.example.toml +++ b/config.example.toml @@ -96,6 +96,8 @@ identifier = "" [service.spotify] # Read listens and loves from a Spotify account +# NOTE: The Spotify API does not allow access to the full listen history, +# but only to recent listens. backend = "spotify" # You need to register an application on https://developer.spotify.com/ # and set the client ID and client secret below. @@ -106,8 +108,6 @@ client-secret = "" [service.spotify-history] # Read listens from a Spotify extended history export -# NOTE: The Spotify API does not allow access to the full listen history, -# but only to recent listens. backend = "spotify-history" # Path to the Spotify extended history archive. This can either point directly # to the "my_spotify_data_extended.zip" ZIP file provided by Spotify or a @@ -135,6 +135,15 @@ backend = "deezer" client-id = "" client-secret = "" +[service.deezer-history] +# Read listens from a Deezer data export. +# You can request a download of all your Deezer data, including the complete +# listen history, in the section "My information" in your Deezer +# "Account settings". +backend = "deezer-history" +# Path to XLSX file provided by Deezer, e.g. "deezer-data_520704045.xlsx". +file-path = "" + [service.lastfm] backend = "lastfm" # Your Last.fm username diff --git a/go.mod b/go.mod index c5c3511..5991ecd 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d github.com/vbauerster/mpb/v8 v8.10.1 + github.com/xuri/excelize/v2 v2.9.1 go.uploadedlobster.com/mbtypes v0.4.0 go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 @@ -52,13 +53,19 @@ require ( github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.9.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tiendc/go-deepcopy v1.6.0 // indirect + github.com/xuri/efp v0.0.1 // indirect + github.com/xuri/nfp v0.0.1 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.38.0 // indirect golang.org/x/image v0.27.0 // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.40.0 // indirect diff --git a/go.sum b/go.sum index 6d34a6d..fefdfa4 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,11 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= +github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= +github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -127,15 +132,21 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d h1:70+Nn7yh+cfeKqqXVTdpneFqXuvrBLyP7U6GVUsjTU4= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d/go.mod h1:D+OWPXeD9owcdcoXATv5YPBGWxxVvn5k98rt5B4wMc4= +github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo= +github.com/tiendc/go-deepcopy v1.6.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I= github.com/vbauerster/mpb/v8 v8.10.1 h1:t/ZFv/NYgoBUy2LrmkD5Vc25r+JhoS4+gRkjVbolO2Y= github.com/vbauerster/mpb/v8 v8.10.1/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= +github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= +github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw= +github.com/xuri/excelize/v2 v2.9.1/go.mod h1:x7L6pKz2dvo9ejrRuD8Lnl98z4JLt0TGAwjhW+EiP8s= +github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q= +github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uploadedlobster.com/mbtypes v0.4.0 h1:D5asCgHsRWufj4Yn5u0IuH2J9z1UuYImYkYIp1Z1Q7s= go.uploadedlobster.com/mbtypes v0.4.0/go.mod h1:Bu1K1Hl77QTAE2Z7QKiW/JAp9KqYWQebkRRfG02dlZM= -go.uploadedlobster.com/musicbrainzws2 v0.15.0 h1:njJeyf1dDwfz2toEHaZSuockVsn1fg+967/tVfLHhwQ= -go.uploadedlobster.com/musicbrainzws2 v0.15.0/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 h1:bir8kas9u0A+T54sfzj3il7SUAV5KQtb5QzDtwvslxI= go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/internal/backends/backends.go b/internal/backends/backends.go index a1cd407..97a78c2 100644 --- a/internal/backends/backends.go +++ b/internal/backends/backends.go @@ -23,6 +23,7 @@ import ( "strings" "go.uploadedlobster.com/scotty/internal/backends/deezer" + "go.uploadedlobster.com/scotty/internal/backends/deezerhistory" "go.uploadedlobster.com/scotty/internal/backends/dump" "go.uploadedlobster.com/scotty/internal/backends/funkwhale" "go.uploadedlobster.com/scotty/internal/backends/jspf" @@ -107,6 +108,7 @@ func GetBackends() BackendList { var knownBackends = map[string]func() models.Backend{ "deezer": func() models.Backend { return &deezer.DeezerApiBackend{} }, + "deezer-history": func() models.Backend { return &deezerhistory.DeezerHistoryBackend{} }, "dump": func() models.Backend { return &dump.DumpBackend{} }, "funkwhale": func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} }, "jspf": func() models.Backend { return &jspf.JSPFBackend{} }, diff --git a/internal/backends/backends_test.go b/internal/backends/backends_test.go index 026e487..b30eb95 100644 --- a/internal/backends/backends_test.go +++ b/internal/backends/backends_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/assert" "go.uploadedlobster.com/scotty/internal/backends" "go.uploadedlobster.com/scotty/internal/backends/deezer" + "go.uploadedlobster.com/scotty/internal/backends/deezerhistory" "go.uploadedlobster.com/scotty/internal/backends/dump" "go.uploadedlobster.com/scotty/internal/backends/funkwhale" "go.uploadedlobster.com/scotty/internal/backends/jspf" @@ -86,6 +87,8 @@ func TestImplementsInterfaces(t *testing.T) { expectInterface[models.LovesExport](t, &deezer.DeezerApiBackend{}) // expectInterface[models.LovesImport](t, &deezer.DeezerApiBackend{}) + expectInterface[models.ListensExport](t, &deezerhistory.DeezerHistoryBackend{}) + expectInterface[models.ListensImport](t, &dump.DumpBackend{}) expectInterface[models.LovesImport](t, &dump.DumpBackend{}) diff --git a/internal/backends/deezer/deezer.go b/internal/backends/deezer/deezer.go index a6eaec2..9d622df 100644 --- a/internal/backends/deezer/deezer.go +++ b/internal/backends/deezer/deezer.go @@ -251,7 +251,7 @@ func (t Track) AsTrack() models.Track { TrackName: t.Title, ReleaseName: t.Album.Title, ArtistNames: []string{t.Artist.Name}, - Duration: time.Duration(t.Duration * int(time.Second)), + Duration: time.Duration(t.Duration) * time.Second, AdditionalInfo: map[string]any{}, } diff --git a/internal/backends/deezerhistory/deezerhistory.go b/internal/backends/deezerhistory/deezerhistory.go new file mode 100644 index 0000000..5574a5d --- /dev/null +++ b/internal/backends/deezerhistory/deezerhistory.go @@ -0,0 +1,206 @@ +/* +Copyright © 2025 Philipp Wolfer + +This file is part of Scotty. + +Scotty is free software: you can redistribute it and/or modify it under the +terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later version. + +Scotty is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +Scotty. If not, see . +*/ + +package deezerhistory + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/xuri/excelize/v2" + "go.uploadedlobster.com/mbtypes" + "go.uploadedlobster.com/scotty/internal/config" + "go.uploadedlobster.com/scotty/internal/i18n" + "go.uploadedlobster.com/scotty/internal/models" +) + +const ( + sheetListeningHistory = "10_listeningHistory" + sheetFavoriteSongs = "8_favoriteSong" +) + +type DeezerHistoryBackend struct { + filePath string +} + +func (b *DeezerHistoryBackend) Name() string { return "deezer-history" } + +func (b *DeezerHistoryBackend) Options() []models.BackendOption { + return []models.BackendOption{{ + Name: "file-path", + Label: i18n.Tr("File path"), + Type: models.String, + Default: "", + }} +} + +func (b *DeezerHistoryBackend) InitConfig(config *config.ServiceConfig) error { + b.filePath = config.GetString("file-path") + return nil +} + +func (b *DeezerHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { + p := models.TransferProgress{ + Export: &models.Progress{}, + } + + rows, err := ReadXLSXSheet(b.filePath, sheetListeningHistory) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + + count := len(rows) - 1 // Exclude the header row + p.Export.TotalItems = count + p.Export.Total = int64(count) + + listens := make(models.ListensList, 0, count) + for i, row := range models.IterExportProgress(rows, &p, progress) { + // Skip header row + if i == 0 { + continue + } + + l, err := RowAsListen(row) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + listens = append(listens, *l) + } + + sort.Sort(listens) + results <- models.ListensResult{Items: listens} + p.Export.Complete() + progress <- p +} + +func (b *DeezerHistoryBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { + p := models.TransferProgress{ + Export: &models.Progress{}, + } + + rows, err := ReadXLSXSheet(b.filePath, sheetFavoriteSongs) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + + count := len(rows) - 1 // Exclude the header row + p.Export.TotalItems = count + p.Export.Total = int64(count) + + love := make(models.LovesList, 0, count) + for i, row := range models.IterExportProgress(rows, &p, progress) { + // Skip header row + if i == 0 { + continue + } + + l, err := RowAsLove(row) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + love = append(love, *l) + } + + sort.Sort(love) + results <- models.LovesResult{Items: love} + p.Export.Complete() + progress <- p +} + +func ReadXLSXSheet(path string, sheet string) ([][]string, error) { + exc, err := excelize.OpenFile(path) + if err != nil { + return nil, err + } + + // Get all the rows in the Sheet1. + return exc.GetRows(sheet) +} + +func RowAsListen(row []string) (*models.Listen, error) { + if len(row) < 9 { + err := fmt.Errorf("Invalid row, expected 9 columns, got %d", len(row)) + return nil, err + } + + listenedAt, err := time.Parse(time.DateTime, row[8]) + if err != nil { + return nil, err + } + listen := models.Listen{ + ListenedAt: listenedAt, + Track: models.Track{ + TrackName: row[0], + ArtistNames: []string{row[1]}, + ReleaseName: row[3], + ISRC: mbtypes.ISRC(row[2]), + AdditionalInfo: map[string]any{ + "music_service": "deezer.com", + }, + }, + } + + if duration, err := strconv.Atoi(row[5]); err == nil { + listen.PlaybackDuration = time.Duration(duration) * time.Second + } + + return &listen, nil +} + +func RowAsLove(row []string) (*models.Love, error) { + if len(row) < 5 { + err := fmt.Errorf("Invalid row, expected 5 columns, got %d", len(row)) + return nil, err + } + + url := row[4] + if !strings.HasPrefix(url, "http://") || !strings.HasPrefix(url, "https") { + url = "https://" + url + } + + love := models.Love{ + Track: models.Track{ + TrackName: row[0], + ArtistNames: []string{row[1]}, + ReleaseName: row[2], + ISRC: mbtypes.ISRC(row[3]), + AdditionalInfo: map[string]any{ + "music_service": "deezer.com", + "origin_url": url, + "deezer_id": url, + }, + }, + } + + return &love, nil +} diff --git a/internal/backends/funkwhale/funkwhale.go b/internal/backends/funkwhale/funkwhale.go index d9632a6..8874c70 100644 --- a/internal/backends/funkwhale/funkwhale.go +++ b/internal/backends/funkwhale/funkwhale.go @@ -220,7 +220,7 @@ func (t Track) AsTrack() models.Track { } if len(t.Uploads) > 0 { - track.Duration = time.Duration(t.Uploads[0].Duration * int(time.Second)) + track.Duration = time.Duration(t.Uploads[0].Duration) * time.Second } return track diff --git a/internal/backends/spotify/spotify.go b/internal/backends/spotify/spotify.go index b00ebba..fbfe821 100644 --- a/internal/backends/spotify/spotify.go +++ b/internal/backends/spotify/spotify.go @@ -258,7 +258,7 @@ func (t Track) AsTrack() models.Track { TrackName: t.Name, ReleaseName: t.Album.Name, ArtistNames: make([]string, 0, len(t.Artists)), - Duration: time.Duration(t.DurationMs * int(time.Millisecond)), + Duration: time.Duration(t.DurationMs) * time.Millisecond, TrackNumber: t.TrackNumber, DiscNumber: t.DiscNumber, ISRC: t.ExternalIDs.ISRC, diff --git a/internal/backends/spotifyhistory/archive.go b/internal/backends/spotifyhistory/archive.go index 1d596bd..2f9a2ec 100644 --- a/internal/backends/spotifyhistory/archive.go +++ b/internal/backends/spotifyhistory/archive.go @@ -21,7 +21,7 @@ import ( "errors" "sort" - "go.uploadedlobster.com/scotty/internal/archive" + "go.uploadedlobster.com/scotty/pkg/archive" ) var historyFileGlobs = []string{ @@ -33,7 +33,7 @@ var historyFileGlobs = []string{ // This can be either the ZIP file as provided by Spotify // or a directory where this was extracted to. type HistoryArchive struct { - backend archive.Archive + backend archive.ArchiveReader } // Open a Spotify history archive from file path. diff --git a/internal/backends/spotifyhistory/models.go b/internal/backends/spotifyhistory/models.go index a2eba23..3efaa38 100644 --- a/internal/backends/spotifyhistory/models.go +++ b/internal/backends/spotifyhistory/models.go @@ -89,7 +89,7 @@ func (i HistoryItem) AsListen() models.Listen { AdditionalInfo: models.AdditionalInfo{}, }, ListenedAt: i.Timestamp, - PlaybackDuration: time.Duration(i.MillisecondsPlayed * int(time.Millisecond)), + PlaybackDuration: time.Duration(i.MillisecondsPlayed) * time.Millisecond, UserName: i.UserName, } if trackURL, err := formatSpotifyUri(i.SpotifyTrackUri); err != nil { diff --git a/internal/backends/subsonic/subsonic.go b/internal/backends/subsonic/subsonic.go index aa1b1e3..f75366d 100644 --- a/internal/backends/subsonic/subsonic.go +++ b/internal/backends/subsonic/subsonic.go @@ -121,7 +121,7 @@ func SongAsLove(song subsonic.Child, username string) models.Love { AdditionalInfo: map[string]any{ "subsonic_id": song.ID, }, - Duration: time.Duration(song.Duration * int(time.Second)), + Duration: time.Duration(song.Duration) * time.Second, }, } diff --git a/internal/listenbrainz/archive.go b/internal/listenbrainz/archive.go index b7b5909..68740aa 100644 --- a/internal/listenbrainz/archive.go +++ b/internal/listenbrainz/archive.go @@ -32,7 +32,7 @@ import ( "time" "github.com/simonfrey/jsonl" - "go.uploadedlobster.com/scotty/internal/archive" + "go.uploadedlobster.com/scotty/pkg/archive" ) // Represents a ListenBrainz export archive. @@ -40,7 +40,7 @@ import ( // The export contains the user's listen history, favorite tracks and // user information. type ExportArchive struct { - backend archive.Archive + backend archive.ArchiveReader } // Open a ListenBrainz archive from file path. diff --git a/internal/archive/archive.go b/pkg/archive/archive.go similarity index 56% rename from internal/archive/archive.go rename to pkg/archive/archive.go index 7714552..41c954f 100644 --- a/internal/archive/archive.go +++ b/pkg/archive/archive.go @@ -27,18 +27,23 @@ THE SOFTWARE. package archive import ( - "archive/zip" "fmt" "io" "io/fs" "os" - "path/filepath" ) -// Generic archive interface. -type Archive interface { - Close() error +// Generic interface to access files inside an archive. +type ArchiveReader interface { + io.Closer + + // Open the file inside the archive identified by the given path. + // The path is relative to the archive's root. + // The caller must call [fs.File.Close] when finished using the file. Open(path string) (fs.File, error) + + // List files inside the archive which satisfy the given glob pattern. + // This method only returns files, not directories. Glob(pattern string) ([]FileInfo, error) } @@ -46,7 +51,7 @@ type Archive interface { // The archive can be a ZIP file or a directory. The implementation // will detect the type of archive and return the appropriate // implementation of the Archive interface. -func OpenArchive(path string) (Archive, error) { +func OpenArchive(path string) (ArchiveReader, error) { fi, err := os.Stat(path) if err != nil { return nil, err @@ -73,10 +78,14 @@ func OpenArchive(path string) (Archive, error) { // Interface for a file that can be opened when needed. type OpenableFile interface { + // Open the file for reading. + // The caller is responsible to call [io.ReadCloser.Close] when + // finished reading the file. Open() (io.ReadCloser, error) } // Generic information about a file inside an archive. +// This provides the filename and allows opening the file for reading. type FileInfo struct { Name string File OpenableFile @@ -90,90 +99,3 @@ type filesystemFile struct { func (f *filesystemFile) Open() (io.ReadCloser, error) { return os.Open(f.path) } - -// An implementation of the archiveBackend interface for zip files. -type zipArchive struct { - zip *zip.ReadCloser -} - -func (a *zipArchive) OpenArchive(path string) error { - zip, err := zip.OpenReader(path) - if err != nil { - return err - } - a.zip = zip - return nil -} - -func (a *zipArchive) Close() error { - if a.zip == nil { - return nil - } - return a.zip.Close() -} - -func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { - result := make([]FileInfo, 0) - for _, file := range a.zip.File { - if matched, err := filepath.Match(pattern, file.Name); matched { - if err != nil { - return nil, err - } - info := FileInfo{ - Name: file.Name, - File: file, - } - result = append(result, info) - } - } - - return result, nil -} - -func (a *zipArchive) Open(path string) (fs.File, error) { - file, err := a.zip.Open(path) - if err != nil { - return nil, err - } - return file, nil -} - -// An implementation of the archiveBackend interface for directories. -type dirArchive struct { - path string - dirFS fs.FS -} - -func (a *dirArchive) OpenArchive(path string) error { - a.path = filepath.Clean(path) - a.dirFS = os.DirFS(path) - return nil -} - -func (a *dirArchive) Close() error { - return nil -} - -// Open opens the named file in the archive. -// [fs.File.Close] must be called to release any associated resources. -func (a *dirArchive) Open(path string) (fs.File, error) { - return a.dirFS.Open(path) -} - -func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { - files, err := fs.Glob(a.dirFS, pattern) - if err != nil { - return nil, err - } - result := make([]FileInfo, 0) - for _, name := range files { - fullPath := filepath.Join(a.path, name) - info := FileInfo{ - Name: name, - File: &filesystemFile{path: fullPath}, - } - result = append(result, info) - } - - return result, nil -} diff --git a/pkg/archive/archive_test.go b/pkg/archive/archive_test.go new file mode 100644 index 0000000..f1bbd07 --- /dev/null +++ b/pkg/archive/archive_test.go @@ -0,0 +1,189 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package archive_test + +import ( + "fmt" + "io" + "log" + "slices" + "testing" + + "go.uploadedlobster.com/scotty/pkg/archive" +) + +func ExampleOpenArchive() { + a, err := archive.OpenArchive("testdata/archive.zip") + if err != nil { + log.Fatal(err) + } + + defer a.Close() + + files, err := a.Glob("a/*.txt") + for _, fi := range files { + fmt.Println(fi.Name) + f, err := fi.File.Open() + if err != nil { + log.Fatal(err) + } + + defer f.Close() + data, err := io.ReadAll(f) + if err != nil { + log.Fatal(err) + } + fmt.Println(string(data)) + } + + // Output: a/1.txt + // a1 +} + +var testArchives = []string{ + "testdata/archive", + "testdata/archive.zip", +} + +func TestGlob(t *testing.T) { + for _, path := range testArchives { + a, err := archive.OpenArchive(path) + if err != nil { + t.Fatal(err) + } + defer a.Close() + + files, err := a.Glob("[ab]/1.txt") + if err != nil { + t.Fatal(err) + } + + if len(files) != 2 { + t.Errorf("Expected 2 files, got %d", len(files)) + } + + expectedName := "b/1.txt" + var fileInfo *archive.FileInfo = nil + for _, file := range files { + if file.Name == expectedName { + fileInfo = &file + } + } + + if fileInfo == nil { + t.Fatalf("Expected file %q to be found", expectedName) + } + + if fileInfo.File == nil { + t.Fatalf("Expected FileInfo to hold an openable File") + } + + f, err := fileInfo.File.Open() + if err != nil { + t.Fatal(err) + } + + expectedData := "b1\n" + data, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if string(data) != expectedData { + fmt.Printf("%s: Expected file content to be %q, got %q", + path, expectedData, string(data)) + } + } +} + +func TestGlobAll(t *testing.T) { + for _, path := range testArchives { + a, err := archive.OpenArchive(path) + if err != nil { + t.Fatal(err) + } + defer a.Close() + + files, err := a.Glob("*/*") + if err != nil { + t.Fatal(err) + } + + filenames := make([]string, 0, len(files)) + for _, f := range files { + fmt.Printf("%v: %v\n", path, f.Name) + filenames = append(filenames, f.Name) + } + + slices.Sort(filenames) + + expectedFilenames := []string{ + "a/1.txt", + "b/1.txt", + "b/2.txt", + } + if !slices.Equal(filenames, expectedFilenames) { + t.Errorf("%s: Expected filenames to be %q, got %q", + path, expectedFilenames, filenames) + } + } +} + +func TestOpen(t *testing.T) { + for _, path := range testArchives { + a, err := archive.OpenArchive(path) + if err != nil { + t.Fatal(err) + } + defer a.Close() + + f, err := a.Open("b/2.txt") + if err != nil { + t.Fatal(err) + } + + expectedData := "b2\n" + data, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if string(data) != expectedData { + fmt.Printf("%s: Expected file content to be %q, got %q", + path, expectedData, string(data)) + } + } +} + +func TestOpenError(t *testing.T) { + for _, path := range testArchives { + a, err := archive.OpenArchive(path) + if err != nil { + t.Fatal(err) + } + defer a.Close() + + _, err = a.Open("b/3.txt") + if err == nil { + t.Errorf("%s: Expected the Open command to fail", path) + } + } +} diff --git a/pkg/archive/dir.go b/pkg/archive/dir.go new file mode 100644 index 0000000..166e70b --- /dev/null +++ b/pkg/archive/dir.go @@ -0,0 +1,77 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package archive + +import ( + "io/fs" + "os" + "path/filepath" +) + +// An implementation of the [ArchiveReader] interface for directories. +type dirArchive struct { + path string + dirFS fs.FS +} + +func (a *dirArchive) OpenArchive(path string) error { + a.path = filepath.Clean(path) + a.dirFS = os.DirFS(path) + return nil +} + +func (a *dirArchive) Close() error { + return nil +} + +// Open opens the named file in the archive. +// [fs.File.Close] must be called to release any associated resources. +func (a *dirArchive) Open(path string) (fs.File, error) { + return a.dirFS.Open(path) +} + +func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { + files, err := fs.Glob(a.dirFS, pattern) + if err != nil { + return nil, err + } + result := make([]FileInfo, 0) + for _, name := range files { + stat, err := fs.Stat(a.dirFS, name) + if err != nil { + return nil, err + } + if stat.IsDir() { + continue + } + + fullPath := filepath.Join(a.path, name) + info := FileInfo{ + Name: name, + File: &filesystemFile{path: fullPath}, + } + result = append(result, info) + } + + return result, nil +} diff --git a/pkg/archive/testdata/archive.zip b/pkg/archive/testdata/archive.zip new file mode 100644 index 0000000..19923f6 Binary files /dev/null and b/pkg/archive/testdata/archive.zip differ diff --git a/pkg/archive/testdata/archive/a/1.txt b/pkg/archive/testdata/archive/a/1.txt new file mode 100644 index 0000000..da0f8ed --- /dev/null +++ b/pkg/archive/testdata/archive/a/1.txt @@ -0,0 +1 @@ +a1 diff --git a/pkg/archive/testdata/archive/b/1.txt b/pkg/archive/testdata/archive/b/1.txt new file mode 100644 index 0000000..c9c6af7 --- /dev/null +++ b/pkg/archive/testdata/archive/b/1.txt @@ -0,0 +1 @@ +b1 diff --git a/pkg/archive/testdata/archive/b/2.txt b/pkg/archive/testdata/archive/b/2.txt new file mode 100644 index 0000000..e6bfff5 --- /dev/null +++ b/pkg/archive/testdata/archive/b/2.txt @@ -0,0 +1 @@ +b2 diff --git a/pkg/archive/zip.go b/pkg/archive/zip.go new file mode 100644 index 0000000..0054cf3 --- /dev/null +++ b/pkg/archive/zip.go @@ -0,0 +1,80 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package archive + +import ( + "archive/zip" + "io/fs" + "path/filepath" +) + +// An implementation of the [ArchiveReader] interface for zip files. +type zipArchive struct { + zip *zip.ReadCloser +} + +func (a *zipArchive) OpenArchive(path string) error { + zip, err := zip.OpenReader(path) + if err != nil { + return err + } + a.zip = zip + return nil +} + +func (a *zipArchive) Close() error { + if a.zip == nil { + return nil + } + return a.zip.Close() +} + +func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { + result := make([]FileInfo, 0) + for _, file := range a.zip.File { + if file.FileInfo().IsDir() { + continue + } + + if matched, err := filepath.Match(pattern, file.Name); matched { + if err != nil { + return nil, err + } + info := FileInfo{ + Name: file.Name, + File: file, + } + result = append(result, info) + } + } + + return result, nil +} + +func (a *zipArchive) Open(path string) (fs.File, error) { + file, err := a.zip.Open(path) + if err != nil { + return nil, err + } + return file, nil +}