Compare commits

...

6 commits

Author SHA1 Message Date
Philipp Wolfer
ed0c31c00f
Update changelog 2025-05-25 15:54:25 +02:00
Philipp Wolfer
0115eca1c6
Minor code cleanup when creation time.Duration 2025-05-25 15:53:01 +02:00
Philipp Wolfer
78a05e9f54
Implemented deezer-history loves export 2025-05-25 15:49:06 +02:00
Philipp Wolfer
e85090fe4a
Implemented deezer-history backend listen import 2025-05-25 15:38:48 +02:00
Philipp Wolfer
1244405747
Moved archive package to public pkg/ 2025-05-25 12:51:36 +02:00
Philipp Wolfer
28c618ffce
Implemented tests and added documentation for archive 2025-05-25 12:46:44 +02:00
23 changed files with 617 additions and 106 deletions

View file

@ -14,6 +14,7 @@
- spotify-history: the parameter to the export archive path has been renamed to
`archive-path`. For backward compatibility the old `dir-path` parameter is
still read.
- deezer-history: new backend to import listens and loves from Deezer data export.
- deezer: fixed endless export loop if the user's listen history was empty.
- dump: it is now possible to specify a file to write the text output to.
- Fixed potential issues with MusicBrainz rate limiting.

View file

@ -120,6 +120,7 @@ The following table lists the available backends and the currently supported fea
Backend | Listens Export | Listens Import | Loves Export | Loves Import
---------------------|----------------|----------------|--------------|-------------
deezer | ✓ | | ✓ | -
deezer-history | ✓ | | ✓ |
funkwhale | ✓ | | ✓ | -
jspf | ✓ | ✓ | ✓ | ✓
lastfm | ✓ | ✓ | ✓ | ✓

View file

@ -96,6 +96,8 @@ identifier = ""
[service.spotify]
# Read listens and loves from a Spotify account
# NOTE: The Spotify API does not allow access to the full listen history,
# but only to recent listens.
backend = "spotify"
# You need to register an application on https://developer.spotify.com/
# and set the client ID and client secret below.
@ -106,8 +108,6 @@ client-secret = ""
[service.spotify-history]
# Read listens from a Spotify extended history export
# NOTE: The Spotify API does not allow access to the full listen history,
# but only to recent listens.
backend = "spotify-history"
# Path to the Spotify extended history archive. This can either point directly
# to the "my_spotify_data_extended.zip" ZIP file provided by Spotify or a
@ -135,6 +135,15 @@ backend = "deezer"
client-id = ""
client-secret = ""
[service.deezer-history]
# Read listens from a Deezer data export.
# You can request a download of all your Deezer data, including the complete
# listen history, in the section "My information" in your Deezer
# "Account settings".
backend = "deezer-history"
# Path to XLSX file provided by Deezer, e.g. "deezer-data_520704045.xlsx".
file-path = ""
[service.lastfm]
backend = "lastfm"
# Your Last.fm username

7
go.mod
View file

@ -22,6 +22,7 @@ require (
github.com/stretchr/testify v1.10.0
github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d
github.com/vbauerster/mpb/v8 v8.10.1
github.com/xuri/excelize/v2 v2.9.1
go.uploadedlobster.com/mbtypes v0.4.0
go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6
@ -52,13 +53,19 @@ require (
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect
github.com/richardlehane/msoleps v1.0.4 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/sagikazarmark/locafero v0.9.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.14.0 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tiendc/go-deepcopy v1.6.0 // indirect
github.com/xuri/efp v0.0.1 // indirect
github.com/xuri/nfp v0.0.1 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.38.0 // indirect
golang.org/x/image v0.27.0 // indirect
golang.org/x/mod v0.24.0 // indirect
golang.org/x/net v0.40.0 // indirect

15
go.sum
View file

@ -97,6 +97,11 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00=
github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@ -127,15 +132,21 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d h1:70+Nn7yh+cfeKqqXVTdpneFqXuvrBLyP7U6GVUsjTU4=
github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d/go.mod h1:D+OWPXeD9owcdcoXATv5YPBGWxxVvn5k98rt5B4wMc4=
github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo=
github.com/tiendc/go-deepcopy v1.6.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I=
github.com/vbauerster/mpb/v8 v8.10.1 h1:t/ZFv/NYgoBUy2LrmkD5Vc25r+JhoS4+gRkjVbolO2Y=
github.com/vbauerster/mpb/v8 v8.10.1/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0=
github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=
github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw=
github.com/xuri/excelize/v2 v2.9.1/go.mod h1:x7L6pKz2dvo9ejrRuD8Lnl98z4JLt0TGAwjhW+EiP8s=
github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q=
github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uploadedlobster.com/mbtypes v0.4.0 h1:D5asCgHsRWufj4Yn5u0IuH2J9z1UuYImYkYIp1Z1Q7s=
go.uploadedlobster.com/mbtypes v0.4.0/go.mod h1:Bu1K1Hl77QTAE2Z7QKiW/JAp9KqYWQebkRRfG02dlZM=
go.uploadedlobster.com/musicbrainzws2 v0.15.0 h1:njJeyf1dDwfz2toEHaZSuockVsn1fg+967/tVfLHhwQ=
go.uploadedlobster.com/musicbrainzws2 v0.15.0/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao=
go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 h1:bir8kas9u0A+T54sfzj3il7SUAV5KQtb5QzDtwvslxI=
go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

View file

@ -23,6 +23,7 @@ import (
"strings"
"go.uploadedlobster.com/scotty/internal/backends/deezer"
"go.uploadedlobster.com/scotty/internal/backends/deezerhistory"
"go.uploadedlobster.com/scotty/internal/backends/dump"
"go.uploadedlobster.com/scotty/internal/backends/funkwhale"
"go.uploadedlobster.com/scotty/internal/backends/jspf"
@ -107,6 +108,7 @@ func GetBackends() BackendList {
var knownBackends = map[string]func() models.Backend{
"deezer": func() models.Backend { return &deezer.DeezerApiBackend{} },
"deezer-history": func() models.Backend { return &deezerhistory.DeezerHistoryBackend{} },
"dump": func() models.Backend { return &dump.DumpBackend{} },
"funkwhale": func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} },
"jspf": func() models.Backend { return &jspf.JSPFBackend{} },

View file

@ -24,6 +24,7 @@ import (
"github.com/stretchr/testify/assert"
"go.uploadedlobster.com/scotty/internal/backends"
"go.uploadedlobster.com/scotty/internal/backends/deezer"
"go.uploadedlobster.com/scotty/internal/backends/deezerhistory"
"go.uploadedlobster.com/scotty/internal/backends/dump"
"go.uploadedlobster.com/scotty/internal/backends/funkwhale"
"go.uploadedlobster.com/scotty/internal/backends/jspf"
@ -86,6 +87,8 @@ func TestImplementsInterfaces(t *testing.T) {
expectInterface[models.LovesExport](t, &deezer.DeezerApiBackend{})
// expectInterface[models.LovesImport](t, &deezer.DeezerApiBackend{})
expectInterface[models.ListensExport](t, &deezerhistory.DeezerHistoryBackend{})
expectInterface[models.ListensImport](t, &dump.DumpBackend{})
expectInterface[models.LovesImport](t, &dump.DumpBackend{})

View file

@ -251,7 +251,7 @@ func (t Track) AsTrack() models.Track {
TrackName: t.Title,
ReleaseName: t.Album.Title,
ArtistNames: []string{t.Artist.Name},
Duration: time.Duration(t.Duration * int(time.Second)),
Duration: time.Duration(t.Duration) * time.Second,
AdditionalInfo: map[string]any{},
}

View file

@ -0,0 +1,206 @@
/*
Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
Scotty is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation, either version 3 of the License, or (at your option) any later version.
Scotty is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
Scotty. If not, see <https://www.gnu.org/licenses/>.
*/
package deezerhistory
import (
"context"
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/xuri/excelize/v2"
"go.uploadedlobster.com/mbtypes"
"go.uploadedlobster.com/scotty/internal/config"
"go.uploadedlobster.com/scotty/internal/i18n"
"go.uploadedlobster.com/scotty/internal/models"
)
const (
sheetListeningHistory = "10_listeningHistory"
sheetFavoriteSongs = "8_favoriteSong"
)
type DeezerHistoryBackend struct {
filePath string
}
func (b *DeezerHistoryBackend) Name() string { return "deezer-history" }
func (b *DeezerHistoryBackend) Options() []models.BackendOption {
return []models.BackendOption{{
Name: "file-path",
Label: i18n.Tr("File path"),
Type: models.String,
Default: "",
}}
}
func (b *DeezerHistoryBackend) InitConfig(config *config.ServiceConfig) error {
b.filePath = config.GetString("file-path")
return nil
}
func (b *DeezerHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
p := models.TransferProgress{
Export: &models.Progress{},
}
rows, err := ReadXLSXSheet(b.filePath, sheetListeningHistory)
if err != nil {
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
count := len(rows) - 1 // Exclude the header row
p.Export.TotalItems = count
p.Export.Total = int64(count)
listens := make(models.ListensList, 0, count)
for i, row := range models.IterExportProgress(rows, &p, progress) {
// Skip header row
if i == 0 {
continue
}
l, err := RowAsListen(row)
if err != nil {
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
listens = append(listens, *l)
}
sort.Sort(listens)
results <- models.ListensResult{Items: listens}
p.Export.Complete()
progress <- p
}
func (b *DeezerHistoryBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
p := models.TransferProgress{
Export: &models.Progress{},
}
rows, err := ReadXLSXSheet(b.filePath, sheetFavoriteSongs)
if err != nil {
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
count := len(rows) - 1 // Exclude the header row
p.Export.TotalItems = count
p.Export.Total = int64(count)
love := make(models.LovesList, 0, count)
for i, row := range models.IterExportProgress(rows, &p, progress) {
// Skip header row
if i == 0 {
continue
}
l, err := RowAsLove(row)
if err != nil {
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
love = append(love, *l)
}
sort.Sort(love)
results <- models.LovesResult{Items: love}
p.Export.Complete()
progress <- p
}
func ReadXLSXSheet(path string, sheet string) ([][]string, error) {
exc, err := excelize.OpenFile(path)
if err != nil {
return nil, err
}
// Get all the rows in the Sheet1.
return exc.GetRows(sheet)
}
func RowAsListen(row []string) (*models.Listen, error) {
if len(row) < 9 {
err := fmt.Errorf("Invalid row, expected 9 columns, got %d", len(row))
return nil, err
}
listenedAt, err := time.Parse(time.DateTime, row[8])
if err != nil {
return nil, err
}
listen := models.Listen{
ListenedAt: listenedAt,
Track: models.Track{
TrackName: row[0],
ArtistNames: []string{row[1]},
ReleaseName: row[3],
ISRC: mbtypes.ISRC(row[2]),
AdditionalInfo: map[string]any{
"music_service": "deezer.com",
},
},
}
if duration, err := strconv.Atoi(row[5]); err == nil {
listen.PlaybackDuration = time.Duration(duration) * time.Second
}
return &listen, nil
}
func RowAsLove(row []string) (*models.Love, error) {
if len(row) < 5 {
err := fmt.Errorf("Invalid row, expected 5 columns, got %d", len(row))
return nil, err
}
url := row[4]
if !strings.HasPrefix(url, "http://") || !strings.HasPrefix(url, "https") {
url = "https://" + url
}
love := models.Love{
Track: models.Track{
TrackName: row[0],
ArtistNames: []string{row[1]},
ReleaseName: row[2],
ISRC: mbtypes.ISRC(row[3]),
AdditionalInfo: map[string]any{
"music_service": "deezer.com",
"origin_url": url,
"deezer_id": url,
},
},
}
return &love, nil
}

View file

@ -220,7 +220,7 @@ func (t Track) AsTrack() models.Track {
}
if len(t.Uploads) > 0 {
track.Duration = time.Duration(t.Uploads[0].Duration * int(time.Second))
track.Duration = time.Duration(t.Uploads[0].Duration) * time.Second
}
return track

View file

@ -258,7 +258,7 @@ func (t Track) AsTrack() models.Track {
TrackName: t.Name,
ReleaseName: t.Album.Name,
ArtistNames: make([]string, 0, len(t.Artists)),
Duration: time.Duration(t.DurationMs * int(time.Millisecond)),
Duration: time.Duration(t.DurationMs) * time.Millisecond,
TrackNumber: t.TrackNumber,
DiscNumber: t.DiscNumber,
ISRC: t.ExternalIDs.ISRC,

View file

@ -21,7 +21,7 @@ import (
"errors"
"sort"
"go.uploadedlobster.com/scotty/internal/archive"
"go.uploadedlobster.com/scotty/pkg/archive"
)
var historyFileGlobs = []string{
@ -33,7 +33,7 @@ var historyFileGlobs = []string{
// This can be either the ZIP file as provided by Spotify
// or a directory where this was extracted to.
type HistoryArchive struct {
backend archive.Archive
backend archive.ArchiveReader
}
// Open a Spotify history archive from file path.

View file

@ -89,7 +89,7 @@ func (i HistoryItem) AsListen() models.Listen {
AdditionalInfo: models.AdditionalInfo{},
},
ListenedAt: i.Timestamp,
PlaybackDuration: time.Duration(i.MillisecondsPlayed * int(time.Millisecond)),
PlaybackDuration: time.Duration(i.MillisecondsPlayed) * time.Millisecond,
UserName: i.UserName,
}
if trackURL, err := formatSpotifyUri(i.SpotifyTrackUri); err != nil {

View file

@ -121,7 +121,7 @@ func SongAsLove(song subsonic.Child, username string) models.Love {
AdditionalInfo: map[string]any{
"subsonic_id": song.ID,
},
Duration: time.Duration(song.Duration * int(time.Second)),
Duration: time.Duration(song.Duration) * time.Second,
},
}

View file

@ -32,7 +32,7 @@ import (
"time"
"github.com/simonfrey/jsonl"
"go.uploadedlobster.com/scotty/internal/archive"
"go.uploadedlobster.com/scotty/pkg/archive"
)
// Represents a ListenBrainz export archive.
@ -40,7 +40,7 @@ import (
// The export contains the user's listen history, favorite tracks and
// user information.
type ExportArchive struct {
backend archive.Archive
backend archive.ArchiveReader
}
// Open a ListenBrainz archive from file path.

View file

@ -27,18 +27,23 @@ THE SOFTWARE.
package archive
import (
"archive/zip"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
)
// Generic archive interface.
type Archive interface {
Close() error
// Generic interface to access files inside an archive.
type ArchiveReader interface {
io.Closer
// Open the file inside the archive identified by the given path.
// The path is relative to the archive's root.
// The caller must call [fs.File.Close] when finished using the file.
Open(path string) (fs.File, error)
// List files inside the archive which satisfy the given glob pattern.
// This method only returns files, not directories.
Glob(pattern string) ([]FileInfo, error)
}
@ -46,7 +51,7 @@ type Archive interface {
// The archive can be a ZIP file or a directory. The implementation
// will detect the type of archive and return the appropriate
// implementation of the Archive interface.
func OpenArchive(path string) (Archive, error) {
func OpenArchive(path string) (ArchiveReader, error) {
fi, err := os.Stat(path)
if err != nil {
return nil, err
@ -73,10 +78,14 @@ func OpenArchive(path string) (Archive, error) {
// Interface for a file that can be opened when needed.
type OpenableFile interface {
// Open the file for reading.
// The caller is responsible to call [io.ReadCloser.Close] when
// finished reading the file.
Open() (io.ReadCloser, error)
}
// Generic information about a file inside an archive.
// This provides the filename and allows opening the file for reading.
type FileInfo struct {
Name string
File OpenableFile
@ -90,90 +99,3 @@ type filesystemFile struct {
func (f *filesystemFile) Open() (io.ReadCloser, error) {
return os.Open(f.path)
}
// An implementation of the archiveBackend interface for zip files.
type zipArchive struct {
zip *zip.ReadCloser
}
func (a *zipArchive) OpenArchive(path string) error {
zip, err := zip.OpenReader(path)
if err != nil {
return err
}
a.zip = zip
return nil
}
func (a *zipArchive) Close() error {
if a.zip == nil {
return nil
}
return a.zip.Close()
}
func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) {
result := make([]FileInfo, 0)
for _, file := range a.zip.File {
if matched, err := filepath.Match(pattern, file.Name); matched {
if err != nil {
return nil, err
}
info := FileInfo{
Name: file.Name,
File: file,
}
result = append(result, info)
}
}
return result, nil
}
func (a *zipArchive) Open(path string) (fs.File, error) {
file, err := a.zip.Open(path)
if err != nil {
return nil, err
}
return file, nil
}
// An implementation of the archiveBackend interface for directories.
type dirArchive struct {
path string
dirFS fs.FS
}
func (a *dirArchive) OpenArchive(path string) error {
a.path = filepath.Clean(path)
a.dirFS = os.DirFS(path)
return nil
}
func (a *dirArchive) Close() error {
return nil
}
// Open opens the named file in the archive.
// [fs.File.Close] must be called to release any associated resources.
func (a *dirArchive) Open(path string) (fs.File, error) {
return a.dirFS.Open(path)
}
func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) {
files, err := fs.Glob(a.dirFS, pattern)
if err != nil {
return nil, err
}
result := make([]FileInfo, 0)
for _, name := range files {
fullPath := filepath.Join(a.path, name)
info := FileInfo{
Name: name,
File: &filesystemFile{path: fullPath},
}
result = append(result, info)
}
return result, nil
}

189
pkg/archive/archive_test.go Normal file
View file

@ -0,0 +1,189 @@
/*
Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package archive_test
import (
"fmt"
"io"
"log"
"slices"
"testing"
"go.uploadedlobster.com/scotty/pkg/archive"
)
func ExampleOpenArchive() {
a, err := archive.OpenArchive("testdata/archive.zip")
if err != nil {
log.Fatal(err)
}
defer a.Close()
files, err := a.Glob("a/*.txt")
for _, fi := range files {
fmt.Println(fi.Name)
f, err := fi.File.Open()
if err != nil {
log.Fatal(err)
}
defer f.Close()
data, err := io.ReadAll(f)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(data))
}
// Output: a/1.txt
// a1
}
var testArchives = []string{
"testdata/archive",
"testdata/archive.zip",
}
func TestGlob(t *testing.T) {
for _, path := range testArchives {
a, err := archive.OpenArchive(path)
if err != nil {
t.Fatal(err)
}
defer a.Close()
files, err := a.Glob("[ab]/1.txt")
if err != nil {
t.Fatal(err)
}
if len(files) != 2 {
t.Errorf("Expected 2 files, got %d", len(files))
}
expectedName := "b/1.txt"
var fileInfo *archive.FileInfo = nil
for _, file := range files {
if file.Name == expectedName {
fileInfo = &file
}
}
if fileInfo == nil {
t.Fatalf("Expected file %q to be found", expectedName)
}
if fileInfo.File == nil {
t.Fatalf("Expected FileInfo to hold an openable File")
}
f, err := fileInfo.File.Open()
if err != nil {
t.Fatal(err)
}
expectedData := "b1\n"
data, err := io.ReadAll(f)
if err != nil {
t.Fatal(err)
}
if string(data) != expectedData {
fmt.Printf("%s: Expected file content to be %q, got %q",
path, expectedData, string(data))
}
}
}
func TestGlobAll(t *testing.T) {
for _, path := range testArchives {
a, err := archive.OpenArchive(path)
if err != nil {
t.Fatal(err)
}
defer a.Close()
files, err := a.Glob("*/*")
if err != nil {
t.Fatal(err)
}
filenames := make([]string, 0, len(files))
for _, f := range files {
fmt.Printf("%v: %v\n", path, f.Name)
filenames = append(filenames, f.Name)
}
slices.Sort(filenames)
expectedFilenames := []string{
"a/1.txt",
"b/1.txt",
"b/2.txt",
}
if !slices.Equal(filenames, expectedFilenames) {
t.Errorf("%s: Expected filenames to be %q, got %q",
path, expectedFilenames, filenames)
}
}
}
func TestOpen(t *testing.T) {
for _, path := range testArchives {
a, err := archive.OpenArchive(path)
if err != nil {
t.Fatal(err)
}
defer a.Close()
f, err := a.Open("b/2.txt")
if err != nil {
t.Fatal(err)
}
expectedData := "b2\n"
data, err := io.ReadAll(f)
if err != nil {
t.Fatal(err)
}
if string(data) != expectedData {
fmt.Printf("%s: Expected file content to be %q, got %q",
path, expectedData, string(data))
}
}
}
func TestOpenError(t *testing.T) {
for _, path := range testArchives {
a, err := archive.OpenArchive(path)
if err != nil {
t.Fatal(err)
}
defer a.Close()
_, err = a.Open("b/3.txt")
if err == nil {
t.Errorf("%s: Expected the Open command to fail", path)
}
}
}

77
pkg/archive/dir.go Normal file
View file

@ -0,0 +1,77 @@
/*
Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package archive
import (
"io/fs"
"os"
"path/filepath"
)
// An implementation of the [ArchiveReader] interface for directories.
type dirArchive struct {
path string
dirFS fs.FS
}
func (a *dirArchive) OpenArchive(path string) error {
a.path = filepath.Clean(path)
a.dirFS = os.DirFS(path)
return nil
}
func (a *dirArchive) Close() error {
return nil
}
// Open opens the named file in the archive.
// [fs.File.Close] must be called to release any associated resources.
func (a *dirArchive) Open(path string) (fs.File, error) {
return a.dirFS.Open(path)
}
func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) {
files, err := fs.Glob(a.dirFS, pattern)
if err != nil {
return nil, err
}
result := make([]FileInfo, 0)
for _, name := range files {
stat, err := fs.Stat(a.dirFS, name)
if err != nil {
return nil, err
}
if stat.IsDir() {
continue
}
fullPath := filepath.Join(a.path, name)
info := FileInfo{
Name: name,
File: &filesystemFile{path: fullPath},
}
result = append(result, info)
}
return result, nil
}

BIN
pkg/archive/testdata/archive.zip vendored Normal file

Binary file not shown.

1
pkg/archive/testdata/archive/a/1.txt vendored Normal file
View file

@ -0,0 +1 @@
a1

1
pkg/archive/testdata/archive/b/1.txt vendored Normal file
View file

@ -0,0 +1 @@
b1

1
pkg/archive/testdata/archive/b/2.txt vendored Normal file
View file

@ -0,0 +1 @@
b2

80
pkg/archive/zip.go Normal file
View file

@ -0,0 +1,80 @@
/*
Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package archive
import (
"archive/zip"
"io/fs"
"path/filepath"
)
// An implementation of the [ArchiveReader] interface for zip files.
type zipArchive struct {
zip *zip.ReadCloser
}
func (a *zipArchive) OpenArchive(path string) error {
zip, err := zip.OpenReader(path)
if err != nil {
return err
}
a.zip = zip
return nil
}
func (a *zipArchive) Close() error {
if a.zip == nil {
return nil
}
return a.zip.Close()
}
func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) {
result := make([]FileInfo, 0)
for _, file := range a.zip.File {
if file.FileInfo().IsDir() {
continue
}
if matched, err := filepath.Match(pattern, file.Name); matched {
if err != nil {
return nil, err
}
info := FileInfo{
Name: file.Name,
File: file,
}
result = append(result, info)
}
}
return result, nil
}
func (a *zipArchive) Open(path string) (fs.File, error) {
file, err := a.zip.Open(path)
if err != nil {
return nil, err
}
return file, nil
}