Spotify extended streaming history exporter

internal/backends/backends.go

@@ -31,6 +31,7 @@ import (
 	"go.uploadedlobster.com/scotty/internal/backends/maloja"
 	"go.uploadedlobster.com/scotty/internal/backends/scrobblerlog"
 	"go.uploadedlobster.com/scotty/internal/backends/spotify"
+	"go.uploadedlobster.com/scotty/internal/backends/spotifyhistory"
 	"go.uploadedlobster.com/scotty/internal/backends/subsonic"
 	"go.uploadedlobster.com/scotty/internal/config"
 	"go.uploadedlobster.com/scotty/internal/i18n"
@@ -104,16 +105,17 @@ func GetBackends() BackendList {
 }
 
 var knownBackends = map[string]func() models.Backend{
-	"deezer":        func() models.Backend { return &deezer.DeezerApiBackend{} },
-	"dump":          func() models.Backend { return &dump.DumpBackend{} },
-	"funkwhale":     func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} },
-	"jspf":          func() models.Backend { return &jspf.JSPFBackend{} },
-	"lastfm":        func() models.Backend { return &lastfm.LastfmApiBackend{} },
-	"listenbrainz":  func() models.Backend { return &listenbrainz.ListenBrainzApiBackend{} },
-	"maloja":        func() models.Backend { return &maloja.MalojaApiBackend{} },
-	"scrobbler-log": func() models.Backend { return &scrobblerlog.ScrobblerLogBackend{} },
-	"spotify":       func() models.Backend { return &spotify.SpotifyApiBackend{} },
-	"subsonic":      func() models.Backend { return &subsonic.SubsonicApiBackend{} },
+	"deezer":          func() models.Backend { return &deezer.DeezerApiBackend{} },
+	"dump":            func() models.Backend { return &dump.DumpBackend{} },
+	"funkwhale":       func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} },
+	"jspf":            func() models.Backend { return &jspf.JSPFBackend{} },
+	"lastfm":          func() models.Backend { return &lastfm.LastfmApiBackend{} },
+	"listenbrainz":    func() models.Backend { return &listenbrainz.ListenBrainzApiBackend{} },
+	"maloja":          func() models.Backend { return &maloja.MalojaApiBackend{} },
+	"scrobbler-log":   func() models.Backend { return &scrobblerlog.ScrobblerLogBackend{} },
+	"spotify":         func() models.Backend { return &spotify.SpotifyApiBackend{} },
+	"spotify-history": func() models.Backend { return &spotifyhistory.SpotifyHistoryBackend{} },
+	"subsonic":        func() models.Backend { return &subsonic.SubsonicApiBackend{} },
 }
 
 func backendWithConfig(config config.ServiceConfig) (models.Backend, error) {

internal/backends/spotifyhistory/models.go

@@ -0,0 +1,110 @@
+/*
+Copyright © 2024 Philipp Wolfer <phw@uploadedlobster.com>
+
+Scotty is free software: you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later version.
+
+Scotty is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+Scotty. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+package spotifyhistory
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"go.uploadedlobster.com/scotty/internal/models"
+)
+
+type StreamingHistory []HistoryItem
+
+type ListenListOptions struct {
+	IgnoreIncognito      bool
+	IgnoreSkipped        bool
+	SkippedMinDurationMs int
+}
+
+type HistoryItem struct {
+	Timestamp                     time.Time `json:"ts"`
+	UserName                      string    `json:"username"`
+	Platform                      string    `json:"platform"`
+	MillisecondsPlayed            int       `json:"ms_played"`
+	ConnCountry                   string    `json:"conn_country"`
+	IpAddrDecrypted               string    `json:"ip_addr_decrypted"`
+	UserAgentDecrypted            string    `json:"user_agent_decrypted"`
+	MasterMetadataTrackName       string    `json:"master_metadata_track_name"`
+	MasterMetadataAlbumArtistName string    `json:"master_metadata_album_artist_name"`
+	MasterMetadataAlbumName       string    `json:"master_metadata_album_album_name"`
+	SpotifyTrackUri               string    `json:"spotify_track_uri"`
+	EpisodeName                   string    `json:"episode_name"`
+	EpisodeShowName               string    `json:"episode_show_name"`
+	SpotifyEpisodeUri             string    `json:"spotify_episode_uri"`
+	ReasonStart                   string    `json:"reason_start"`
+	ReasonEnd                     string    `json:"reason_end"`
+	Shuffle                       bool      `json:"shuffle"`
+	Skipped                       bool      `json:"skipped"`
+	Offline                       bool      `json:"offline"`
+	OfflineTimestamp              int       `json:"offline_timestamp"`
+	IncognitoMode                 bool      `json:"incognito_mode"`
+}
+
+func (j *StreamingHistory) Read(in io.Reader) error {
+	bytes, err := io.ReadAll(in)
+	if err != nil {
+		return err
+	}
+	err = json.Unmarshal(bytes, j)
+	return err
+}
+
+func (h *StreamingHistory) AsListenList(opt ListenListOptions) models.ListensList {
+	listens := make(models.ListensList, 0, len(*h))
+	for _, item := range *h {
+		if item.MasterMetadataTrackName == "" ||
+			(opt.IgnoreIncognito && item.IncognitoMode) ||
+			(opt.IgnoreSkipped && item.Skipped) ||
+			(item.Skipped && item.MillisecondsPlayed < opt.SkippedMinDurationMs) {
+			continue
+		}
+		listens = append(listens, item.AsListen())
+	}
+	return listens
+}
+
+func (i HistoryItem) AsListen() models.Listen {
+	listen := models.Listen{
+		Track: models.Track{
+			TrackName:      i.MasterMetadataTrackName,
+			ReleaseName:    i.MasterMetadataAlbumName,
+			ArtistNames:    []string{i.MasterMetadataAlbumArtistName},
+			AdditionalInfo: models.AdditionalInfo{},
+		},
+		ListenedAt:       i.Timestamp,
+		PlaybackDuration: time.Duration(i.MillisecondsPlayed * int(time.Millisecond)),
+		UserName:         i.UserName,
+	}
+	if trackUrl, err := formatSpotifyUri(i.SpotifyTrackUri); err != nil {
+		listen.AdditionalInfo["spotify_id"] = trackUrl
+	}
+	return listen
+}
+
+// Returns a Spotify ID like "spotify:track:5jzma6gCzYtKB1DbEwFZKH" into an
+// URL like "https://open.spotify.com/track/5jzma6gCzYtKB1DbEwFZKH"
+func formatSpotifyUri(id string) (string, error) {
+	parts := strings.Split(id, ":")
+	if len(parts) == 3 && parts[0] == "spotify" {
+		return fmt.Sprintf("https://opem.spotify.com/%s/%s", parts[1], parts[2]), nil
+	} else {
+		return "", fmt.Errorf("Invalid Spotify ID \"%v\"", id)
+	}
+}

internal/backends/spotifyhistory/spotifyhistory.go

@@ -0,0 +1,118 @@
+/*
+Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+
+This file is part of Scotty.
+
+Scotty is free software: you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later version.
+
+Scotty is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+Scotty. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+package spotifyhistory
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"slices"
+	"sort"
+	"time"
+
+	"go.uploadedlobster.com/scotty/internal/config"
+	"go.uploadedlobster.com/scotty/internal/i18n"
+	"go.uploadedlobster.com/scotty/internal/models"
+)
+
+const historyFileGlob = "Streaming_History_Audio_*.json"
+
+type SpotifyHistoryBackend struct {
+	dirPath              string
+	ignoreIncognito      bool
+	ignoreSkipped        bool
+	skippedMinDurationMs int
+}
+
+func (b *SpotifyHistoryBackend) Name() string { return "spotify-history" }
+
+func (b *SpotifyHistoryBackend) Options() []models.BackendOption {
+	return []models.BackendOption{{
+		Name:  "dir-path",
+		Label: i18n.Tr("Directory path"),
+		Type:  models.String,
+	}, {
+		Name:    "ignore-incognito",
+		Label:   i18n.Tr("Ignore listens in incognito mode"),
+		Type:    models.Bool,
+		Default: "true",
+	}, {
+		Name:    "ignore-skipped",
+		Label:   i18n.Tr("Ignore skipped listens"),
+		Type:    models.Bool,
+		Default: "false",
+	}}
+}
+
+func (b *SpotifyHistoryBackend) FromConfig(config *config.ServiceConfig) models.Backend {
+	b.dirPath = config.GetString("dir-path")
+	b.ignoreIncognito = config.GetBool("ignore-incognito", true)
+	b.ignoreSkipped = config.GetBool("ignore-skipped", false)
+	b.skippedMinDurationMs = 30000
+	return b
+}
+
+func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
+	defer close(results)
+
+	files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob))
+	if err != nil {
+		progress <- models.Progress{}.Complete()
+		results <- models.ListensResult{Error: err}
+		return
+	}
+
+	slices.Sort(files)
+	fileCount := int64(len(files))
+	p := models.Progress{Total: fileCount}
+	for i, filePath := range files {
+		history, err := readHistoryFile(filePath)
+		if err != nil {
+			progress <- models.Progress{}.Complete()
+			results <- models.ListensResult{Error: err}
+			return
+		}
+		listens := history.AsListenList(ListenListOptions{
+			IgnoreIncognito:      b.ignoreIncognito,
+			IgnoreSkipped:        b.ignoreSkipped,
+			SkippedMinDurationMs: b.skippedMinDurationMs,
+		})
+		sort.Sort(listens)
+		results <- models.ListensResult{Items: listens}
+		p.Elapsed = int64(i)
+		progress <- p
+	}
+
+	progress <- p.Complete()
+}
+
+func readHistoryFile(filePath string) (StreamingHistory, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	defer file.Close()
+	history := StreamingHistory{}
+	err = history.Read(file)
+	if err != nil {
+		return nil, err
+	}
+
+	return history, nil
+}