Compare commits

...

14 commits

Author SHA1 Message Date
Philipp Wolfer
39b31fc664
Update changelog 2025-04-29 13:01:54 +02:00
Philipp Wolfer
1516a3a9d6
scrobblerlog: renamed setting include-skipped to ignore-skipped
This makes the setting consistent with the similar setting for spotify
2025-04-29 12:57:28 +02:00
Philipp Wolfer
82858315fa
Disable Linux 386 builds
Compilaton fails with latest gorm
2025-04-29 11:44:04 +02:00
Philipp Wolfer
e135ea5fa9
Update goreleaser config file format 2025-04-29 11:43:42 +02:00
Philipp Wolfer
597914e6db
Announce new releases to Go Module Index 2025-04-29 11:15:49 +02:00
Philipp Wolfer
c817480809
Updated Weblate CI secret and fixed build 2025-04-29 11:12:28 +02:00
Philipp Wolfer
47486ff659
Update weblate configuration 2025-04-29 11:05:37 +02:00
Philipp Wolfer
159f486cdc
Upgrade musicbrainzws2 2025-04-29 10:32:59 +02:00
Philipp Wolfer
b104c2bc42
scrobblerlog: fixed listen export progress 2025-04-29 10:10:32 +02:00
Philipp Wolfer
ed191d2f15
scrobblerlog: Allow configuring fallback time zone
Fixes #6
2025-04-29 10:05:40 +02:00
Philipp Wolfer
0f4b04c641
Renamed Backend.FromConfig to Backend.InitConfig and added error handling 2025-04-29 10:03:28 +02:00
Philipp Wolfer
aad542850a
scrobblerlog: Use specific Record type
This makes the interface more generic and easier to reuse in other
projects.
2025-04-29 09:18:57 +02:00
Philipp Wolfer
aeb3a56982
Moved scrobblerlog parsing to separate package 2025-04-29 08:44:31 +02:00
Philipp Wolfer
69665bc286
scrobblerlog: consider timezone from parsed file 2025-04-29 07:54:27 +02:00
32 changed files with 525 additions and 336 deletions

View file

@ -5,10 +5,11 @@ packages:
- hut
- weblate-wlc
secrets:
- 2a17e258-3e99-4093-9527-832c350d9c53
- 0e2ad815-6c46-4cea-878e-70fc33f71e77
oauth: pages.sr.ht/PAGES:RW
tasks:
- weblate-update: |
cd scotty
wlc --format text pull scotty
- test: |
cd scotty
@ -28,5 +29,15 @@ tasks:
- publish-redirect: |
# Update redirect on https://go.uploadedlobster.com/scotty
./scotty/pages/publish.sh
# Skip releasing if this is not a tagged release
- only-tags: |
cd scotty
GIT_REF=$(git describe --always)
[[ "$GIT_REF" =~ ^v[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || complete-build
- announce-release: |
# Announce new release to Go Module Index
cd scotty
VERSION=$(git describe --exact-match)
curl "https://proxy.golang.org/go.uploadedlobster.com/scotty/@v/${VERSION}.info"
artifacts:
- scotty/dist/artifacts.tar

View file

@ -6,7 +6,7 @@
# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
# vim: set ts=2 sw=2 tw=0 fo=cnqoj
version: 1
version: 2
before:
hooks:
@ -21,6 +21,8 @@ builds:
- windows
- darwin
ignore:
- goos: linux
goarch: "386"
- goos: windows
goarch: "386"
@ -28,7 +30,7 @@ universal_binaries:
- replace: true
archives:
- format: tar.gz
- formats: ['tar.gz']
# this name template makes the OS and Arch compatible with the results of `uname`.
name_template: >-
{{ .ProjectName }}-{{ .Version }}_
@ -42,7 +44,7 @@ archives:
# use zip for windows archives
format_overrides:
- goos: windows
format: zip
formats: ['zip']
files:
- COPYING
- README.md

3
.weblate Normal file
View file

@ -0,0 +1,3 @@
[weblate]
url = https://translate.uploadedlobster.com/api/
translation = scotty/app

View file

@ -7,6 +7,12 @@
- ListenBrainz: log missing recording MBID on love import
- Subsonic: support OpenSubsonic fields for recording MBID and genres (#5)
- Subsonic: fixed progress for loves export
- scrobblerlog: add "time-zone" config option (#6).
- scrobblerlog: fixed progress for listen export
- scrobblerlog: renamed setting `include-skipped` to `ignore-skipped`.
Note: 386 builds for Linux are not available with this release due to an
incompatibility with latest version of gorm.
## 0.4.1 - 2024-09-16

View file

@ -56,11 +56,18 @@ backend = "scrobbler-log"
# The file path to the .scrobbler.log file. Relative paths are resolved against
# the current working directory when running scotty.
file-path = "./.scrobbler.log"
# If true, reading listens from the file also returns listens marked as "skipped"
include-skipped = true
# If true (default), ignore listens marked as skipped.
ignore-skipped = true
# If true (default), new listens will be appended to the existing file. Set to
# false to overwrite the file and create a new scrobbler log on every run.
append = true
# Specify the time zone of the listens in the scrobbler log. While the log files
# are supposed to contain Unix timestamps, which are always in UTC, the player
# writing the log might not be time zone aware. This can cause the timestamps
# to be in a different time zone. Use the time-zone setting to specify a
# different time zone, e.g. "Europe/Berlin" or "America/New_York".
# The default is UTC.
time-zone = "UTC"
[service.jspf]
# Write listens and loves to JSPF playlist files (https://xspf.org/jspf)
@ -98,9 +105,9 @@ dir-path = "./my_spotify_data_extended/Spotify Extended Streaming Histor
ignore-incognito = true
# If true, ignore listens marked as skipped. Default is false.
ignore-skipped = false
# Only consider skipped listens with a playback duration longer than this number
# of seconds. Default is 30 seconds. If ignore-skipped is set to false this
# setting has no effect.
# Only consider skipped listens with a playback duration longer than or equal to
# this number of seconds. Default is 30 seconds. If ignore-skipped is enabled
# this setting has no effect.
ignore-min-duration-seconds = 30
[service.deezer]

2
go.mod
View file

@ -22,7 +22,7 @@ require (
github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d
github.com/vbauerster/mpb/v8 v8.9.3
go.uploadedlobster.com/mbtypes v0.4.0
go.uploadedlobster.com/musicbrainzws2 v0.13.1
go.uploadedlobster.com/musicbrainzws2 v0.14.0
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0
golang.org/x/oauth2 v0.29.0
golang.org/x/text v0.24.0

4
go.sum
View file

@ -132,8 +132,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uploadedlobster.com/mbtypes v0.4.0 h1:D5asCgHsRWufj4Yn5u0IuH2J9z1UuYImYkYIp1Z1Q7s=
go.uploadedlobster.com/mbtypes v0.4.0/go.mod h1:Bu1K1Hl77QTAE2Z7QKiW/JAp9KqYWQebkRRfG02dlZM=
go.uploadedlobster.com/musicbrainzws2 v0.13.1 h1:34GKI7l9eTCyh9ozNOHmlwAAUTDK9WVRsFZK5trxcwQ=
go.uploadedlobster.com/musicbrainzws2 v0.13.1/go.mod h1:TVln70Fzp/++fw0/jCP1xXwgilVwDkzTwRbV8GwUYLA=
go.uploadedlobster.com/musicbrainzws2 v0.14.0 h1:YaEtxNwLSNT1gzFipQ4XlaThNfXjBpzzb4I6WhIeUwg=
go.uploadedlobster.com/musicbrainzws2 v0.14.0/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=

View file

@ -123,7 +123,11 @@ func backendWithConfig(config config.ServiceConfig) (models.Backend, error) {
if err != nil {
return nil, err
}
return backend.FromConfig(&config), nil
err = backend.InitConfig(&config)
if err != nil {
return nil, err
}
return backend, nil
}
func ImplementsInterface[T interface{}](backend *models.Backend) (bool, string) {

View file

@ -49,10 +49,10 @@ func (b *DeezerApiBackend) Options() []models.BackendOption {
}}
}
func (b *DeezerApiBackend) FromConfig(config *config.ServiceConfig) models.Backend {
func (b *DeezerApiBackend) InitConfig(config *config.ServiceConfig) error {
b.clientId = config.GetString("client-id")
b.clientSecret = config.GetString("client-secret")
return b
return nil
}
func (b *DeezerApiBackend) OAuth2Strategy(redirectUrl *url.URL) auth.OAuth2Strategy {

View file

@ -35,13 +35,14 @@ var (
testTrack []byte
)
func TestFromConfig(t *testing.T) {
func TestInitConfig(t *testing.T) {
c := viper.New()
c.Set("client-id", "someclientid")
c.Set("client-secret", "someclientsecret")
service := config.NewServiceConfig("test", c)
backend := (&deezer.DeezerApiBackend{}).FromConfig(&service)
assert.IsType(t, &deezer.DeezerApiBackend{}, backend)
backend := deezer.DeezerApiBackend{}
err := backend.InitConfig(&service)
assert.NoError(t, err)
}
func TestListenAsListen(t *testing.T) {

View file

@ -29,8 +29,8 @@ func (b *DumpBackend) Name() string { return "dump" }
func (b *DumpBackend) Options() []models.BackendOption { return nil }
func (b *DumpBackend) FromConfig(config *config.ServiceConfig) models.Backend {
return b
func (b *DumpBackend) InitConfig(config *config.ServiceConfig) error {
return nil
}
func (b *DumpBackend) StartImport() error { return nil }

View file

@ -51,13 +51,13 @@ func (b *FunkwhaleApiBackend) Options() []models.BackendOption {
}}
}
func (b *FunkwhaleApiBackend) FromConfig(config *config.ServiceConfig) models.Backend {
func (b *FunkwhaleApiBackend) InitConfig(config *config.ServiceConfig) error {
b.client = NewClient(
config.GetString("server-url"),
config.GetString("token"),
)
b.username = config.GetString("username")
return b
return nil
}
func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {

View file

@ -27,12 +27,13 @@ import (
"go.uploadedlobster.com/scotty/internal/config"
)
func TestFromConfig(t *testing.T) {
func TestInitConfig(t *testing.T) {
c := viper.New()
c.Set("token", "thetoken")
service := config.NewServiceConfig("test", c)
backend := (&funkwhale.FunkwhaleApiBackend{}).FromConfig(&service)
assert.IsType(t, &funkwhale.FunkwhaleApiBackend{}, backend)
backend := funkwhale.FunkwhaleApiBackend{}
err := backend.InitConfig(&service)
assert.NoError(t, err)
}
func TestFunkwhaleListeningAsListen(t *testing.T) {

View file

@ -60,7 +60,7 @@ func (b *JSPFBackend) Options() []models.BackendOption {
}}
}
func (b *JSPFBackend) FromConfig(config *config.ServiceConfig) models.Backend {
func (b *JSPFBackend) InitConfig(config *config.ServiceConfig) error {
b.filePath = config.GetString("file-path")
b.append = config.GetBool("append", true)
b.playlist = jspf.Playlist{
@ -75,7 +75,7 @@ func (b *JSPFBackend) FromConfig(config *config.ServiceConfig) models.Backend {
},
},
}
return b
return nil
}
func (b *JSPFBackend) StartImport() error {

View file

@ -26,13 +26,14 @@ import (
"go.uploadedlobster.com/scotty/internal/config"
)
func TestFromConfig(t *testing.T) {
func TestInitConfig(t *testing.T) {
c := viper.New()
c.Set("file-path", "/foo/bar.jspf")
c.Set("title", "My Playlist")
c.Set("username", "outsidecontext")
c.Set("identifier", "http://example.com/playlist1")
service := config.NewServiceConfig("test", c)
backend := (&jspf.JSPFBackend{}).FromConfig(&service)
assert.IsType(t, &jspf.JSPFBackend{}, backend)
backend := jspf.JSPFBackend{}
err := backend.InitConfig(&service)
assert.NoError(t, err)
}

View file

@ -61,12 +61,12 @@ func (b *LastfmApiBackend) Options() []models.BackendOption {
}}
}
func (b *LastfmApiBackend) FromConfig(config *config.ServiceConfig) models.Backend {
func (b *LastfmApiBackend) InitConfig(config *config.ServiceConfig) error {
clientId := config.GetString("client-id")
clientSecret := config.GetString("client-secret")
b.client = lastfm.New(clientId, clientSecret)
b.username = config.GetString("username")
return b
return nil
}
func (b *LastfmApiBackend) StartImport() error { return nil }

View file

@ -56,13 +56,17 @@ func (b *ListenBrainzApiBackend) Options() []models.BackendOption {
}}
}
func (b *ListenBrainzApiBackend) FromConfig(config *config.ServiceConfig) models.Backend {
func (b *ListenBrainzApiBackend) InitConfig(config *config.ServiceConfig) error {
b.client = NewClient(config.GetString("token"))
b.mbClient = *musicbrainzws2.NewClient(version.AppName, version.AppVersion)
b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{
Name: version.AppName,
Version: version.AppVersion,
URL: version.AppURL,
})
b.client.MaxResults = MaxItemsPerGet
b.username = config.GetString("username")
b.checkDuplicates = config.GetBool("check-duplicate-listens", false)
return b
return nil
}
func (b *ListenBrainzApiBackend) StartImport() error { return nil }

View file

@ -28,12 +28,13 @@ import (
"go.uploadedlobster.com/scotty/internal/config"
)
func TestFromConfig(t *testing.T) {
func TestInitConfig(t *testing.T) {
c := viper.New()
c.Set("token", "thetoken")
service := config.NewServiceConfig("test", c)
backend := (&listenbrainz.ListenBrainzApiBackend{}).FromConfig(&service)
assert.IsType(t, &listenbrainz.ListenBrainzApiBackend{}, backend)
backend := listenbrainz.ListenBrainzApiBackend{}
err := backend.InitConfig(&service)
assert.NoError(t, err)
}
func TestListenBrainzListenAsListen(t *testing.T) {

View file

@ -51,13 +51,13 @@ func (b *MalojaApiBackend) Options() []models.BackendOption {
}}
}
func (b *MalojaApiBackend) FromConfig(config *config.ServiceConfig) models.Backend {
func (b *MalojaApiBackend) InitConfig(config *config.ServiceConfig) error {
b.client = NewClient(
config.GetString("server-url"),
config.GetString("token"),
)
b.nofix = config.GetBool("nofix", false)
return b
return nil
}
func (b *MalojaApiBackend) StartImport() error { return nil }

View file

@ -26,12 +26,13 @@ import (
"go.uploadedlobster.com/scotty/internal/config"
)
func TestFromConfig(t *testing.T) {
func TestInitConfig(t *testing.T) {
c := viper.New()
c.Set("token", "thetoken")
service := config.NewServiceConfig("test", c)
backend := (&maloja.MalojaApiBackend{}).FromConfig(&service)
assert.IsType(t, &maloja.MalojaApiBackend{}, backend)
backend := maloja.MalojaApiBackend{}
err := backend.InitConfig(&service)
assert.NoError(t, err)
}
func TestScrobbleAsListen(t *testing.T) {

View file

@ -1,211 +0,0 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package scrobblerlog
import (
"bufio"
"encoding/csv"
"fmt"
"io"
"strconv"
"strings"
"time"
"go.uploadedlobster.com/mbtypes"
"go.uploadedlobster.com/scotty/internal/models"
)
type ScrobblerLog struct {
Timezone string
Client string
Listens models.ListensList
}
func Parse(data io.Reader, includeSkipped bool) (ScrobblerLog, error) {
result := ScrobblerLog{
Listens: make(models.ListensList, 0),
}
reader := bufio.NewReader(data)
err := ReadHeader(reader, &result)
if err != nil {
return result, err
}
tsvReader := csv.NewReader(reader)
tsvReader.Comma = '\t'
// Row length is often flexible
tsvReader.FieldsPerRecord = -1
for {
// A row is:
// artistName releaseName trackName trackNumber duration rating timestamp recordingMBID
row, err := tsvReader.Read()
if err == io.EOF {
break
} else if err != nil {
return result, err
}
// fmt.Printf("row: %v\n", row)
// We consider only the last field (recording MBID) optional
if len(row) < 7 {
line, _ := tsvReader.FieldPos(0)
return result, fmt.Errorf("invalid record in scrobblerlog line %v", line)
}
rating := row[5]
if !includeSkipped && rating == "S" {
continue
}
client := strings.Split(result.Client, " ")[0]
listen, err := rowToListen(row, client)
if err != nil {
return result, err
}
result.Listens = append(result.Listens, listen)
}
return result, nil
}
func Write(data io.Writer, listens models.ListensList) (lastTimestamp time.Time, err error) {
tsvWriter := csv.NewWriter(data)
tsvWriter.Comma = '\t'
for _, listen := range listens {
if listen.ListenedAt.Unix() > lastTimestamp.Unix() {
lastTimestamp = listen.ListenedAt
}
// A row is:
// artistName releaseName trackName trackNumber duration rating timestamp recordingMBID
rating, ok := listen.AdditionalInfo["rockbox_rating"].(string)
if !ok || rating == "" {
rating = "L"
}
err = tsvWriter.Write([]string{
listen.ArtistName(),
listen.ReleaseName,
listen.TrackName,
strconv.Itoa(listen.TrackNumber),
strconv.Itoa(int(listen.Duration.Seconds())),
rating,
strconv.Itoa(int(listen.ListenedAt.Unix())),
string(listen.RecordingMBID),
})
}
tsvWriter.Flush()
return
}
func ReadHeader(reader *bufio.Reader, log *ScrobblerLog) error {
// Skip header
for i := 0; i < 3; i++ {
line, _, err := reader.ReadLine()
if err != nil {
return err
}
if len(line) == 0 || line[0] != '#' {
err = fmt.Errorf("unexpected header (line %v)", i)
} else {
text := string(line)
if i == 0 && !strings.HasPrefix(text, "#AUDIOSCROBBLER/1") {
err = fmt.Errorf("not a scrobbler log file")
}
timezone, found := strings.CutPrefix(text, "#TZ/")
if found {
log.Timezone = timezone
}
client, found := strings.CutPrefix(text, "#CLIENT/")
if found {
log.Client = client
}
}
if err != nil {
return err
}
}
return nil
}
func WriteHeader(writer io.Writer, log *ScrobblerLog) error {
headers := []string{
"#AUDIOSCROBBLER/1.1\n",
"#TZ/" + log.Timezone + "\n",
"#CLIENT/" + log.Client + "\n",
}
for _, line := range headers {
_, err := writer.Write([]byte(line))
if err != nil {
return err
}
}
return nil
}
func rowToListen(row []string, client string) (models.Listen, error) {
var listen models.Listen
trackNumber, err := strconv.Atoi(row[3])
if err != nil {
return listen, err
}
duration, err := strconv.Atoi(row[4])
if err != nil {
return listen, err
}
timestamp, err := strconv.Atoi(row[6])
if err != nil {
return listen, err
}
listen = models.Listen{
Track: models.Track{
ArtistNames: []string{row[0]},
ReleaseName: row[1],
TrackName: row[2],
TrackNumber: trackNumber,
Duration: time.Duration(duration * int(time.Second)),
AdditionalInfo: models.AdditionalInfo{
"rockbox_rating": row[5],
"media_player": client,
},
},
ListenedAt: time.Unix(int64(timestamp), 0),
}
if len(row) > 7 {
listen.Track.RecordingMBID = mbtypes.MBID(row[7])
}
return listen, nil
}

View file

@ -18,21 +18,25 @@ package scrobblerlog
import (
"bufio"
"fmt"
"os"
"sort"
"strings"
"time"
"go.uploadedlobster.com/scotty/internal/config"
"go.uploadedlobster.com/scotty/internal/i18n"
"go.uploadedlobster.com/scotty/internal/models"
"go.uploadedlobster.com/scotty/pkg/scrobblerlog"
)
type ScrobblerLogBackend struct {
filePath string
includeSkipped bool
append bool
file *os.File
log ScrobblerLog
filePath string
ignoreSkipped bool
append bool
file *os.File
timezone *time.Location
log scrobblerlog.ScrobblerLog
}
func (b *ScrobblerLogBackend) Name() string { return "scrobbler-log" }
@ -43,26 +47,39 @@ func (b *ScrobblerLogBackend) Options() []models.BackendOption {
Label: i18n.Tr("File path"),
Type: models.String,
}, {
Name: "include-skipped",
Label: i18n.Tr("Include skipped listens"),
Type: models.Bool,
Name: "ignore-skipped",
Label: i18n.Tr("Ignore skipped listens"),
Type: models.Bool,
Default: "true",
}, {
Name: "append",
Label: i18n.Tr("Append to file"),
Type: models.Bool,
Default: "true",
}, {
Name: "time-zone",
Label: i18n.Tr("Specify a time zone for the listen timestamps"),
Type: models.String,
}}
}
func (b *ScrobblerLogBackend) FromConfig(config *config.ServiceConfig) models.Backend {
func (b *ScrobblerLogBackend) InitConfig(config *config.ServiceConfig) error {
b.filePath = config.GetString("file-path")
b.includeSkipped = config.GetBool("include-skipped", false)
b.ignoreSkipped = config.GetBool("ignore-skipped", true)
b.append = config.GetBool("append", true)
b.log = ScrobblerLog{
Timezone: "UNKNOWN",
Client: "Rockbox unknown $Revision$",
timezone := config.GetString("time-zone")
if timezone != "" {
location, err := time.LoadLocation(timezone)
if err != nil {
return fmt.Errorf("Invalid time-zone %q: %w", timezone, err)
}
b.log.FallbackTimezone = location
}
return b
b.log = scrobblerlog.ScrobblerLog{
TZ: scrobblerlog.TZ_UTC,
Client: "Rockbox unknown $Revision$",
}
return nil
}
func (b *ScrobblerLogBackend) StartImport() error {
@ -88,7 +105,7 @@ func (b *ScrobblerLogBackend) StartImport() error {
} else {
// Verify existing file is a scrobbler log
reader := bufio.NewReader(file)
if err = ReadHeader(reader, &b.log); err != nil {
if err = b.log.ReadHeader(reader); err != nil {
file.Close()
return err
}
@ -99,7 +116,7 @@ func (b *ScrobblerLogBackend) StartImport() error {
}
if !b.append {
if err = WriteHeader(file, &b.log); err != nil {
if err = b.log.WriteHeader(file); err != nil {
file.Close()
return err
}
@ -124,21 +141,29 @@ func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results c
defer file.Close()
log, err := Parse(file, b.includeSkipped)
err = b.log.Parse(file, b.ignoreSkipped)
if err != nil {
progress <- models.Progress{}.Complete()
results <- models.ListensResult{Error: err}
return
}
listens := log.Listens.NewerThan(oldestTimestamp)
sort.Sort(listens)
progress <- models.Progress{Elapsed: int64(len(listens))}.Complete()
listens := make(models.ListensList, 0, len(b.log.Records))
client := strings.Split(b.log.Client, " ")[0]
for _, record := range b.log.Records {
listens = append(listens, recordToListen(record, client))
}
sort.Sort(listens.NewerThan(oldestTimestamp))
progress <- models.Progress{Total: int64(len(listens))}.Complete()
results <- models.ListensResult{Items: listens}
}
func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
lastTimestamp, err := Write(b.file, export.Items)
records := make([]scrobblerlog.Record, len(export.Items))
for i, listen := range export.Items {
records[i] = listenToRecord(listen)
}
lastTimestamp, err := b.log.Append(b.file, records)
if err != nil {
return importResult, err
}
@ -149,3 +174,42 @@ func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importR
return importResult, nil
}
func recordToListen(record scrobblerlog.Record, client string) models.Listen {
return models.Listen{
ListenedAt: record.Timestamp,
Track: models.Track{
ArtistNames: []string{record.ArtistName},
ReleaseName: record.AlbumName,
TrackName: record.TrackName,
TrackNumber: record.TrackNumber,
Duration: record.Duration,
RecordingMBID: record.MusicBrainzRecordingID,
AdditionalInfo: models.AdditionalInfo{
"rockbox_rating": record.Rating,
"media_player": client,
},
},
}
}
func listenToRecord(listen models.Listen) scrobblerlog.Record {
var rating scrobblerlog.Rating
rockboxRating, ok := listen.AdditionalInfo["rockbox_rating"].(string)
if !ok || rockboxRating == "" {
rating = scrobblerlog.RATING_LISTENED
} else {
rating = scrobblerlog.Rating(rating)
}
return scrobblerlog.Record{
ArtistName: listen.ArtistName(),
AlbumName: listen.ReleaseName,
TrackName: listen.TrackName,
TrackNumber: listen.TrackNumber,
Duration: listen.Duration,
Rating: rating,
Timestamp: listen.ListenedAt,
MusicBrainzRecordingID: listen.RecordingMBID,
}
}

View file

@ -25,10 +25,21 @@ import (
"go.uploadedlobster.com/scotty/internal/config"
)
func TestFromConfig(t *testing.T) {
func TestInitConfig(t *testing.T) {
c := viper.New()
c.Set("token", "thetoken")
service := config.NewServiceConfig("test", c)
backend := (&scrobblerlog.ScrobblerLogBackend{}).FromConfig(&service)
assert.IsType(t, &scrobblerlog.ScrobblerLogBackend{}, backend)
backend := scrobblerlog.ScrobblerLogBackend{}
err := backend.InitConfig(&service)
assert.NoError(t, err)
}
func TestInitConfigInvalidTimezone(t *testing.T) {
c := viper.New()
configuredTimezone := "Invalid/Timezone"
c.Set("time-zone", configuredTimezone)
service := config.NewServiceConfig("test", c)
backend := scrobblerlog.ScrobblerLogBackend{}
err := backend.InitConfig(&service)
assert.ErrorContains(t, err, `Invalid time-zone "Invalid/Timezone"`)
}

View file

@ -52,10 +52,10 @@ func (b *SpotifyApiBackend) Options() []models.BackendOption {
}}
}
func (b *SpotifyApiBackend) FromConfig(config *config.ServiceConfig) models.Backend {
func (b *SpotifyApiBackend) InitConfig(config *config.ServiceConfig) error {
b.clientId = config.GetString("client-id")
b.clientSecret = config.GetString("client-secret")
return b
return nil
}
func (b *SpotifyApiBackend) OAuth2Strategy(redirectUrl *url.URL) auth.OAuth2Strategy {

View file

@ -38,13 +38,14 @@ var (
testTrack []byte
)
func TestFromConfig(t *testing.T) {
func TestInitConfig(t *testing.T) {
c := viper.New()
c.Set("client-id", "someclientid")
c.Set("client-secret", "someclientsecret")
service := config.NewServiceConfig("test", c)
backend := (&spotify.SpotifyApiBackend{}).FromConfig(&service)
assert.IsType(t, &spotify.SpotifyApiBackend{}, backend)
backend := spotify.SpotifyApiBackend{}
err := backend.InitConfig(&service)
assert.NoError(t, err)
}
func TestSpotifyListenAsListen(t *testing.T) {

View file

@ -64,12 +64,12 @@ func (b *SpotifyHistoryBackend) Options() []models.BackendOption {
}}
}
func (b *SpotifyHistoryBackend) FromConfig(config *config.ServiceConfig) models.Backend {
func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error {
b.dirPath = config.GetString("dir-path")
b.ignoreIncognito = config.GetBool("ignore-incognito", true)
b.ignoreSkipped = config.GetBool("ignore-skipped", false)
b.skippedMinSeconds = config.GetInt("ignore-min-duration-seconds", 30)
return b
return nil
}
func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {

View file

@ -52,7 +52,7 @@ func (b *SubsonicApiBackend) Options() []models.BackendOption {
}}
}
func (b *SubsonicApiBackend) FromConfig(config *config.ServiceConfig) models.Backend {
func (b *SubsonicApiBackend) InitConfig(config *config.ServiceConfig) error {
b.client = subsonic.Client{
Client: &http.Client{},
BaseUrl: config.GetString("server-url"),
@ -60,7 +60,7 @@ func (b *SubsonicApiBackend) FromConfig(config *config.ServiceConfig) models.Bac
ClientName: version.AppName,
}
b.password = config.GetString("token")
return b
return nil
}
func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {

View file

@ -27,13 +27,14 @@ import (
"go.uploadedlobster.com/scotty/internal/config"
)
func TestFromConfig(t *testing.T) {
func TestInitConfig(t *testing.T) {
c := viper.New()
c.Set("server-url", "https://subsonic.example.com")
c.Set("token", "thetoken")
service := config.NewServiceConfig("test", c)
backend := (&subsonic.SubsonicApiBackend{}).FromConfig(&service)
assert.IsType(t, &subsonic.SubsonicApiBackend{}, backend)
backend := subsonic.SubsonicApiBackend{}
err := backend.InitConfig(&service)
assert.NoError(t, err)
}
func TestSongToLove(t *testing.T) {

View file

@ -30,7 +30,7 @@ type Backend interface {
Name() string
// Initialize the backend from a config.
FromConfig(config *config.ServiceConfig) Backend
InitConfig(config *config.ServiceConfig) error
// Return configuration options
Options() []BackendOption

View file

@ -18,6 +18,7 @@ package version
const (
AppName = "scotty"
AppVersion = "0.4.1"
AppURL = "https://git.sr.ht/~phw/scotty/"
)
func UserAgent() string {

264
pkg/scrobblerlog/parser.go Normal file
View file

@ -0,0 +1,264 @@
/*
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
// Package to parse and writer .scrobbler.log files as written by Rockbox.
//
// See
// - https://www.rockbox.org/wiki/LastFMLog
// - https://git.rockbox.org/cgit/rockbox.git/tree/apps/plugins/lastfm_scrobbler.c
package scrobblerlog
import (
"bufio"
"encoding/csv"
"fmt"
"io"
"strconv"
"strings"
"time"
"go.uploadedlobster.com/mbtypes"
)
// TZInfo is the timezone information in the header of the scrobbler log file.
// It can be "UTC" or "UNKNOWN", if the device writing the scrobbler log file
// knows the time, but not the timezone.
type TZInfo string
const (
TZ_UNKNOWN TZInfo = "UNKNOWN"
TZ_UTC TZInfo = "UTC"
)
// L if listened at least 50% or S if skipped
type Rating string
const (
RATING_LISTENED Rating = "L"
RATING_SKIPPED Rating = "S"
)
// A single entry of a track in the scrobbler log file.
type Record struct {
ArtistName string
AlbumName string
TrackName string
TrackNumber int
Duration time.Duration
Rating Rating
Timestamp time.Time
MusicBrainzRecordingID mbtypes.MBID
}
// Represents a scrobbler log file.
type ScrobblerLog struct {
TZ TZInfo
Client string
Records []Record
// Timezone to be used for timestamps in the log file,
// if TZ is set to [TZ_UNKNOWN].
FallbackTimezone *time.Location
}
func (l *ScrobblerLog) Parse(data io.Reader, ignoreSkipped bool) error {
l.Records = make([]Record, 0)
reader := bufio.NewReader(data)
err := l.ReadHeader(reader)
if err != nil {
return err
}
tsvReader := csv.NewReader(reader)
tsvReader.Comma = '\t'
// Row length is often flexible
tsvReader.FieldsPerRecord = -1
for {
// A row is:
// artistName releaseName trackName trackNumber duration rating timestamp recordingMBID
row, err := tsvReader.Read()
if err == io.EOF {
break
} else if err != nil {
return err
}
// fmt.Printf("row: %v\n", row)
// We consider only the last field (recording MBID) optional
if len(row) < 7 {
line, _ := tsvReader.FieldPos(0)
return fmt.Errorf("invalid record in scrobblerlog line %v", line)
}
record, err := l.rowToRecord(row)
if err != nil {
return err
}
if ignoreSkipped && record.Rating == RATING_SKIPPED {
continue
}
l.Records = append(l.Records, record)
}
return nil
}
func (l *ScrobblerLog) Append(data io.Writer, records []Record) (lastTimestamp time.Time, err error) {
tsvWriter := csv.NewWriter(data)
tsvWriter.Comma = '\t'
for _, record := range records {
if record.Timestamp.After(lastTimestamp) {
lastTimestamp = record.Timestamp
}
// A row is:
// artistName releaseName trackName trackNumber duration rating timestamp recordingMBID
err = tsvWriter.Write([]string{
record.ArtistName,
record.AlbumName,
record.TrackName,
strconv.Itoa(record.TrackNumber),
strconv.Itoa(int(record.Duration.Seconds())),
string(record.Rating),
strconv.FormatInt(record.Timestamp.Unix(), 10),
string(record.MusicBrainzRecordingID),
})
}
tsvWriter.Flush()
return
}
func (l *ScrobblerLog) ReadHeader(reader *bufio.Reader) error {
// Skip header
for i := 0; i < 3; i++ {
line, _, err := reader.ReadLine()
if err != nil {
return err
}
if len(line) == 0 || line[0] != '#' {
err = fmt.Errorf("unexpected header (line %v)", i)
} else {
text := string(line)
if i == 0 && !strings.HasPrefix(text, "#AUDIOSCROBBLER/1") {
err = fmt.Errorf("not a scrobbler log file")
}
// The timezone can be set to "UTC" or "UNKNOWN", if the device writing
// the log knows the time, but not the timezone.
timezone, found := strings.CutPrefix(text, "#TZ/")
if found {
l.TZ = TZInfo(timezone)
continue
}
client, found := strings.CutPrefix(text, "#CLIENT/")
if found {
l.Client = client
continue
}
}
if err != nil {
return err
}
}
return nil
}
func (l *ScrobblerLog) WriteHeader(writer io.Writer) error {
headers := []string{
"#AUDIOSCROBBLER/1.1\n",
"#TZ/" + string(l.TZ) + "\n",
"#CLIENT/" + l.Client + "\n",
}
for _, line := range headers {
_, err := writer.Write([]byte(line))
if err != nil {
return err
}
}
return nil
}
func (l ScrobblerLog) rowToRecord(row []string) (Record, error) {
var record Record
trackNumber, err := strconv.Atoi(row[3])
if err != nil {
return record, err
}
duration, err := strconv.Atoi(row[4])
if err != nil {
return record, err
}
timestamp, err := strconv.ParseInt(row[6], 10, 64)
if err != nil {
return record, err
}
var timezone *time.Location = nil
if l.TZ == TZ_UNKNOWN {
timezone = l.FallbackTimezone
}
record = Record{
ArtistName: row[0],
AlbumName: row[1],
TrackName: row[2],
TrackNumber: trackNumber,
Duration: time.Duration(duration) * time.Second,
Rating: Rating(row[5]),
Timestamp: timeFromLocalTimestamp(timestamp, timezone),
}
if len(row) > 7 {
record.MusicBrainzRecordingID = mbtypes.MBID(row[7])
}
return record, nil
}
// Convert a Unix timestamp to a [time.Time] object, but treat the timestamp
// as being in the given location's timezone instead of UTC.
// If location is nil, the timestamp is returned as UTC.
func timeFromLocalTimestamp(timestamp int64, location *time.Location) time.Time {
t := time.Unix(timestamp, 0)
// The time is now in UTC. Get the offset to the requested timezone
// and shift the time accordingly.
if location != nil {
_, offset := t.In(location).Zone()
if offset != 0 {
t = t.Add(time.Duration(offset) * time.Second)
}
}
return t
}

View file

@ -31,8 +31,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uploadedlobster.com/mbtypes"
"go.uploadedlobster.com/scotty/internal/backends/scrobblerlog"
"go.uploadedlobster.com/scotty/internal/models"
"go.uploadedlobster.com/scotty/pkg/scrobblerlog"
)
var testScrobblerLog = `#AUDIOSCROBBLER/1.1
@ -48,67 +47,84 @@ Teeth Agency You Don't Have To Live In Pain Wolfs Jam 2 107 L 1260359404 1262bea
func TestParser(t *testing.T) {
assert := assert.New(t)
data := bytes.NewBufferString(testScrobblerLog)
result, err := scrobblerlog.Parse(data, true)
result := scrobblerlog.ScrobblerLog{}
err := result.Parse(data, false)
require.NoError(t, err)
assert.Equal("UNKNOWN", result.Timezone)
assert.Equal(scrobblerlog.TZ_UNKNOWN, result.TZ)
assert.Equal("Rockbox sansaclipplus $Revision$", result.Client)
assert.Len(result.Listens, 5)
listen1 := result.Listens[0]
assert.Equal("Özcan Deniz", listen1.ArtistName())
assert.Equal("Ses ve Ayrilik", listen1.ReleaseName)
assert.Equal("Sevdanin rengi (sipacik) byMrTurkey", listen1.TrackName)
assert.Equal(5, listen1.TrackNumber)
assert.Equal(time.Duration(306*time.Second), listen1.Duration)
assert.Equal("L", listen1.AdditionalInfo["rockbox_rating"])
assert.Equal(time.Unix(1260342084, 0), listen1.ListenedAt)
assert.Equal(mbtypes.MBID(""), listen1.RecordingMBID)
listen4 := result.Listens[3]
assert.Equal("S", listen4.AdditionalInfo["rockbox_rating"])
assert.Equal(mbtypes.MBID("385ba9e9-626d-4750-a607-58e541dca78e"), listen4.RecordingMBID)
assert.Len(result.Records, 5)
record1 := result.Records[0]
assert.Equal("Özcan Deniz", record1.ArtistName)
assert.Equal("Ses ve Ayrilik", record1.AlbumName)
assert.Equal("Sevdanin rengi (sipacik) byMrTurkey", record1.TrackName)
assert.Equal(5, record1.TrackNumber)
assert.Equal(time.Duration(306*time.Second), record1.Duration)
assert.Equal(scrobblerlog.RATING_LISTENED, record1.Rating)
assert.Equal(time.Unix(1260342084, 0), record1.Timestamp)
assert.Equal(mbtypes.MBID(""), record1.MusicBrainzRecordingID)
record4 := result.Records[3]
assert.Equal(scrobblerlog.RATING_SKIPPED, record4.Rating)
assert.Equal(mbtypes.MBID("385ba9e9-626d-4750-a607-58e541dca78e"),
record4.MusicBrainzRecordingID)
}
func TestParserExcludeSkipped(t *testing.T) {
func TestParserIgnoreSkipped(t *testing.T) {
assert := assert.New(t)
data := bytes.NewBufferString(testScrobblerLog)
result, err := scrobblerlog.Parse(data, false)
result := scrobblerlog.ScrobblerLog{}
err := result.Parse(data, true)
require.NoError(t, err)
assert.Len(result.Listens, 4)
listen4 := result.Listens[3]
assert.Equal("L", listen4.AdditionalInfo["rockbox_rating"])
assert.Equal(mbtypes.MBID("1262beaf-19f8-4534-b9ed-7eef9ca8e83f"), listen4.RecordingMBID)
assert.Len(result.Records, 4)
record4 := result.Records[3]
assert.Equal(scrobblerlog.RATING_LISTENED, record4.Rating)
assert.Equal(mbtypes.MBID("1262beaf-19f8-4534-b9ed-7eef9ca8e83f"),
record4.MusicBrainzRecordingID)
}
func TestWrite(t *testing.T) {
func TestParserFallbackTimezone(t *testing.T) {
assert := assert.New(t)
data := bytes.NewBufferString(testScrobblerLog)
result := scrobblerlog.ScrobblerLog{
FallbackTimezone: time.FixedZone("UTC+2", 7200),
}
err := result.Parse(data, false)
require.NoError(t, err)
record1 := result.Records[0]
assert.Equal(
time.Unix(1260342084, 0).Add(2*time.Hour),
record1.Timestamp,
)
}
func TestAppend(t *testing.T) {
assert := assert.New(t)
data := make([]byte, 0, 10)
buffer := bytes.NewBuffer(data)
log := scrobblerlog.ScrobblerLog{
Timezone: "Unknown",
Client: "Rockbox foo $Revision$",
Listens: []models.Listen{
{
ListenedAt: time.Unix(1699572072, 0),
Track: models.Track{
ArtistNames: []string{"Prinzhorn Dance School"},
ReleaseName: "Home Economics",
TrackName: "Reign",
TrackNumber: 1,
Duration: 271 * time.Second,
RecordingMBID: mbtypes.MBID("b59cf4e7-caee-4019-a844-79d2c58d4dff"),
AdditionalInfo: models.AdditionalInfo{"rockbox_rating": "L"},
},
},
TZ: scrobblerlog.TZ_UNKNOWN,
Client: "Rockbox foo $Revision$",
}
records := []scrobblerlog.Record{
{
ArtistName: "Prinzhorn Dance School",
AlbumName: "Home Economics",
TrackName: "Reign",
TrackNumber: 1,
Duration: 271 * time.Second,
Rating: scrobblerlog.RATING_LISTENED,
Timestamp: time.Unix(1699572072, 0),
MusicBrainzRecordingID: mbtypes.MBID("b59cf4e7-caee-4019-a844-79d2c58d4dff"),
},
}
err := scrobblerlog.WriteHeader(buffer, &log)
err := log.WriteHeader(buffer)
require.NoError(t, err)
lastTimestamp, err := scrobblerlog.Write(buffer, log.Listens)
lastTimestamp, err := log.Append(buffer, records)
require.NoError(t, err)
result := buffer.String()
lines := strings.Split(result, "\n")
assert.Equal(5, len(lines))
assert.Equal("#AUDIOSCROBBLER/1.1", lines[0])
assert.Equal("#TZ/Unknown", lines[1])
assert.Equal("#TZ/UNKNOWN", lines[1])
assert.Equal("#CLIENT/Rockbox foo $Revision$", lines[2])
assert.Equal(
"Prinzhorn Dance School\tHome Economics\tReign\t1\t271\tL\t1699572072\tb59cf4e7-caee-4019-a844-79d2c58d4dff",
@ -121,9 +137,9 @@ func TestReadHeader(t *testing.T) {
data := bytes.NewBufferString(testScrobblerLog)
reader := bufio.NewReader(data)
log := scrobblerlog.ScrobblerLog{}
err := scrobblerlog.ReadHeader(reader, &log)
err := log.ReadHeader(reader)
assert.NoError(t, err)
assert.Equal(t, log.Timezone, "UNKNOWN")
assert.Equal(t, log.TZ, scrobblerlog.TZ_UNKNOWN)
assert.Equal(t, log.Client, "Rockbox sansaclipplus $Revision$")
assert.Empty(t, log.Listens)
assert.Empty(t, log.Records)
}