From e85090fe4a1143c5b1c3734d06cb29a9f35669c5 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 25 May 2025 15:34:56 +0200 Subject: [PATCH] Implemented deezer-history backend listen import --- README.md | 1 + config.example.toml | 13 +- go.mod | 7 + go.sum | 15 +- internal/backends/backends.go | 2 + internal/backends/backends_test.go | 3 + .../backends/deezerhistory/deezerhistory.go | 134 ++++++++++++++++++ 7 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 internal/backends/deezerhistory/deezerhistory.go diff --git a/README.md b/README.md index b10a030..6f8e378 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,7 @@ The following table lists the available backends and the currently supported fea Backend | Listens Export | Listens Import | Loves Export | Loves Import ---------------------|----------------|----------------|--------------|------------- deezer | ✓ | ⨯ | ✓ | - +deezer-history | ✓ | ⨯ | - | ⨯ funkwhale | ✓ | ⨯ | ✓ | - jspf | ✓ | ✓ | ✓ | ✓ lastfm | ✓ | ✓ | ✓ | ✓ diff --git a/config.example.toml b/config.example.toml index 3acdf88..91d5318 100644 --- a/config.example.toml +++ b/config.example.toml @@ -96,6 +96,8 @@ identifier = "" [service.spotify] # Read listens and loves from a Spotify account +# NOTE: The Spotify API does not allow access to the full listen history, +# but only to recent listens. backend = "spotify" # You need to register an application on https://developer.spotify.com/ # and set the client ID and client secret below. @@ -106,8 +108,6 @@ client-secret = "" [service.spotify-history] # Read listens from a Spotify extended history export -# NOTE: The Spotify API does not allow access to the full listen history, -# but only to recent listens. backend = "spotify-history" # Path to the Spotify extended history archive. This can either point directly # to the "my_spotify_data_extended.zip" ZIP file provided by Spotify or a @@ -135,6 +135,15 @@ backend = "deezer" client-id = "" client-secret = "" +[service.deezer-history] +# Read listens from a Deezer data export. +# You can request a download of all your Deezer data, including the complete +# listen history, in the section "My information" in your Deezer +# "Account settings". +backend = "deezer-history" +# Path to XLSX file provided by Deezer, e.g. "deezer-data_520704045.xlsx". +file-path = "" + [service.lastfm] backend = "lastfm" # Your Last.fm username diff --git a/go.mod b/go.mod index c5c3511..5991ecd 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d github.com/vbauerster/mpb/v8 v8.10.1 + github.com/xuri/excelize/v2 v2.9.1 go.uploadedlobster.com/mbtypes v0.4.0 go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 @@ -52,13 +53,19 @@ require ( github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.9.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tiendc/go-deepcopy v1.6.0 // indirect + github.com/xuri/efp v0.0.1 // indirect + github.com/xuri/nfp v0.0.1 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.38.0 // indirect golang.org/x/image v0.27.0 // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.40.0 // indirect diff --git a/go.sum b/go.sum index 6d34a6d..fefdfa4 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,11 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= +github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= +github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -127,15 +132,21 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d h1:70+Nn7yh+cfeKqqXVTdpneFqXuvrBLyP7U6GVUsjTU4= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d/go.mod h1:D+OWPXeD9owcdcoXATv5YPBGWxxVvn5k98rt5B4wMc4= +github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo= +github.com/tiendc/go-deepcopy v1.6.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I= github.com/vbauerster/mpb/v8 v8.10.1 h1:t/ZFv/NYgoBUy2LrmkD5Vc25r+JhoS4+gRkjVbolO2Y= github.com/vbauerster/mpb/v8 v8.10.1/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= +github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= +github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw= +github.com/xuri/excelize/v2 v2.9.1/go.mod h1:x7L6pKz2dvo9ejrRuD8Lnl98z4JLt0TGAwjhW+EiP8s= +github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q= +github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uploadedlobster.com/mbtypes v0.4.0 h1:D5asCgHsRWufj4Yn5u0IuH2J9z1UuYImYkYIp1Z1Q7s= go.uploadedlobster.com/mbtypes v0.4.0/go.mod h1:Bu1K1Hl77QTAE2Z7QKiW/JAp9KqYWQebkRRfG02dlZM= -go.uploadedlobster.com/musicbrainzws2 v0.15.0 h1:njJeyf1dDwfz2toEHaZSuockVsn1fg+967/tVfLHhwQ= -go.uploadedlobster.com/musicbrainzws2 v0.15.0/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 h1:bir8kas9u0A+T54sfzj3il7SUAV5KQtb5QzDtwvslxI= go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/internal/backends/backends.go b/internal/backends/backends.go index a1cd407..97a78c2 100644 --- a/internal/backends/backends.go +++ b/internal/backends/backends.go @@ -23,6 +23,7 @@ import ( "strings" "go.uploadedlobster.com/scotty/internal/backends/deezer" + "go.uploadedlobster.com/scotty/internal/backends/deezerhistory" "go.uploadedlobster.com/scotty/internal/backends/dump" "go.uploadedlobster.com/scotty/internal/backends/funkwhale" "go.uploadedlobster.com/scotty/internal/backends/jspf" @@ -107,6 +108,7 @@ func GetBackends() BackendList { var knownBackends = map[string]func() models.Backend{ "deezer": func() models.Backend { return &deezer.DeezerApiBackend{} }, + "deezer-history": func() models.Backend { return &deezerhistory.DeezerHistoryBackend{} }, "dump": func() models.Backend { return &dump.DumpBackend{} }, "funkwhale": func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} }, "jspf": func() models.Backend { return &jspf.JSPFBackend{} }, diff --git a/internal/backends/backends_test.go b/internal/backends/backends_test.go index 026e487..b30eb95 100644 --- a/internal/backends/backends_test.go +++ b/internal/backends/backends_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/assert" "go.uploadedlobster.com/scotty/internal/backends" "go.uploadedlobster.com/scotty/internal/backends/deezer" + "go.uploadedlobster.com/scotty/internal/backends/deezerhistory" "go.uploadedlobster.com/scotty/internal/backends/dump" "go.uploadedlobster.com/scotty/internal/backends/funkwhale" "go.uploadedlobster.com/scotty/internal/backends/jspf" @@ -86,6 +87,8 @@ func TestImplementsInterfaces(t *testing.T) { expectInterface[models.LovesExport](t, &deezer.DeezerApiBackend{}) // expectInterface[models.LovesImport](t, &deezer.DeezerApiBackend{}) + expectInterface[models.ListensExport](t, &deezerhistory.DeezerHistoryBackend{}) + expectInterface[models.ListensImport](t, &dump.DumpBackend{}) expectInterface[models.LovesImport](t, &dump.DumpBackend{}) diff --git a/internal/backends/deezerhistory/deezerhistory.go b/internal/backends/deezerhistory/deezerhistory.go new file mode 100644 index 0000000..fb8a464 --- /dev/null +++ b/internal/backends/deezerhistory/deezerhistory.go @@ -0,0 +1,134 @@ +/* +Copyright © 2025 Philipp Wolfer + +This file is part of Scotty. + +Scotty is free software: you can redistribute it and/or modify it under the +terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later version. + +Scotty is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +Scotty. If not, see . +*/ + +package deezerhistory + +import ( + "context" + "fmt" + "sort" + "strconv" + "time" + + "github.com/xuri/excelize/v2" + "go.uploadedlobster.com/mbtypes" + "go.uploadedlobster.com/scotty/internal/config" + "go.uploadedlobster.com/scotty/internal/i18n" + "go.uploadedlobster.com/scotty/internal/models" +) + +const ( + sheetListeningHistory = "10_listeningHistory" + sheetfavoriteSongs = "8_favoriteSong" +) + +type DeezerHistoryBackend struct { + filePath string +} + +func (b *DeezerHistoryBackend) Name() string { return "deezer-history" } + +func (b *DeezerHistoryBackend) Options() []models.BackendOption { + return []models.BackendOption{{ + Name: "file-path", + Label: i18n.Tr("File path"), + Type: models.String, + Default: "", + }} +} + +func (b *DeezerHistoryBackend) InitConfig(config *config.ServiceConfig) error { + b.filePath = config.GetString("file-path") + return nil +} + +func (b *DeezerHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { + p := models.TransferProgress{ + Export: &models.Progress{}, + } + + rows, err := ReadXLSXSheet(b.filePath, sheetListeningHistory) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + + count := len(rows) - 1 // Exclude the header row + p.Export.TotalItems = count + p.Export.Total = int64(count) + + listens := make(models.ListensList, 0, count) + for i, row := range models.IterExportProgress(rows, &p, progress) { + // Skip header row + if i == 0 { + continue + } + + l, err := RowAsListen(row) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + listens = append(listens, *l) + } + + sort.Sort(listens) + results <- models.ListensResult{Items: listens} + p.Export.Complete() + progress <- p +} + +func ReadXLSXSheet(path string, sheet string) ([][]string, error) { + exc, err := excelize.OpenFile(path) + if err != nil { + return nil, err + } + + // Get all the rows in the Sheet1. + return exc.GetRows(sheetListeningHistory) +} + +func RowAsListen(row []string) (*models.Listen, error) { + if len(row) < 9 { + err := fmt.Errorf("Invalid row, expected 9 columns, got %d", len(row)) + return nil, err + } + + listenedAt, err := time.Parse(time.DateTime, row[8]) + if err != nil { + return nil, err + } + listen := models.Listen{ + ListenedAt: listenedAt, + Track: models.Track{ + TrackName: row[0], + ArtistNames: []string{row[1]}, + ReleaseName: row[3], + ISRC: mbtypes.ISRC(row[2]), + }, + } + + if duration, err := strconv.Atoi(row[5]); err == nil { + listen.PlaybackDuration = time.Duration(duration) * time.Second + } + + return &listen, nil +}