From cf8a6d2ab6960ccd252b8ec71ef748d07169c99f Mon Sep 17 00:00:00 2001
From: Philipp Wolfer <ph.wolfer@gmail.com>
Date: Sat, 11 Nov 2023 17:08:23 +0100
Subject: [PATCH] Tests for reading / writing scrobbler log
---
backends/scrobblerlog.go | 174 ++-------------------
backends/scrobblerlog/parser.go | 217 +++++++++++++++++++++++++++
backends/scrobblerlog/parser_test.go | 114 ++++++++++++++
3 files changed, 344 insertions(+), 161 deletions(-)
create mode 100644 backends/scrobblerlog/parser.go
create mode 100644 backends/scrobblerlog/parser_test.go
diff --git a/backends/scrobblerlog.go b/backends/scrobblerlog.go
index 01fe8a8..efda58f 100644
--- a/backends/scrobblerlog.go
+++ b/backends/scrobblerlog.go
@@ -22,17 +22,11 @@ THE SOFTWARE.
package backends
import (
- "bufio"
- "encoding/csv"
- "errors"
- "fmt"
- "io"
"os"
- "strconv"
- "strings"
"time"
"github.com/spf13/viper"
+ "go.uploadedlobster.com/scotty/backends/scrobblerlog"
"go.uploadedlobster.com/scotty/models"
)
@@ -55,51 +49,12 @@ func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time) ([]models.
defer file.Close()
- reader := bufio.NewReader(file)
- client, err := readHeader(reader)
+ result, err := scrobblerlog.Parse(file, b.includeSkipped)
if err != nil {
return nil, err
}
- tsvReader := csv.NewReader(reader)
- tsvReader.Comma = '\t'
- // Row length is often flexible
- tsvReader.FieldsPerRecord = -1
-
- listens := make([]models.Listen, 0)
- for {
- // A row is:
- // artistName releaseName trackName trackNumber duration rating timestamp recordingMbid
- row, err := tsvReader.Read()
- if err == io.EOF {
- break
- } else if err != nil {
- return nil, err
- }
-
- // fmt.Printf("row: %v\n", row)
-
- // We consider only the last field (recording MBID) optional
- if len(row) < 7 {
- line, _ := tsvReader.FieldPos(0)
- return nil, errors.New(fmt.Sprintf(
- "Invalid record in %s line %v", b.filePath, line))
- }
-
- rating := row[5]
- if !b.includeSkipped && rating == "S" {
- continue
- }
-
- listen, err := rowToListen(row, client)
- if err != nil {
- return nil, err
- }
-
- listens = append(listens, listen)
- }
-
- return listens, nil
+ return result.Listens, nil
}
func (b ScrobblerLogBackend) ImportListens(listens []models.Listen, oldestTimestamp time.Time) (ImportResult, error) {
@@ -115,122 +70,19 @@ func (b ScrobblerLogBackend) ImportListens(listens []models.Listen, oldestTimest
defer file.Close()
- err = writeHeader(file)
+ log := scrobblerlog.ScrobblerLog{
+ Timezone: "UNKNOWN",
+ Client: "Rockbox unknown $Revision$",
+ Listens: listens,
+ }
+
+ lastTimestamp, err := scrobblerlog.Write(file, &log)
+
if err != nil {
return result, err
}
- tsvWriter := csv.NewWriter(file)
- tsvWriter.Comma = '\t'
-
- for _, listen := range listens {
- result.Count += 1
- if listen.ListenedAt.Unix() > result.LastTimestamp.Unix() {
- result.LastTimestamp = listen.ListenedAt
- }
-
- // A row is:
- // artistName releaseName trackName trackNumber duration rating timestamp recordingMbid
- rating, ok := listen.AdditionalInfo["rockbox_rating"].(string)
- if !ok || rating == "" {
- rating = "L"
- }
- tsvWriter.Write([]string{
- listen.ArtistName(),
- listen.ReleaseName,
- listen.TrackName,
- strconv.Itoa(listen.TrackNumber),
- strconv.Itoa(int(listen.Duration.Seconds())),
- rating,
- strconv.Itoa(int(listen.ListenedAt.Unix())),
- string(listen.RecordingMbid),
- })
- }
-
- tsvWriter.Flush()
-
+ result.LastTimestamp = lastTimestamp
+ result.Count = len(listens)
return result, nil
}
-
-func readHeader(reader *bufio.Reader) (client string, err error) {
- // Skip header
- for i := 0; i < 3; i++ {
- line, _, err := reader.ReadLine()
- if err != nil {
- return client, err
- }
-
- if len(line) == 0 || line[0] != '#' {
- err = errors.New(fmt.Sprintf("Unexpected header (line %v)", i))
- } else {
- text := string(line)
- if i == 0 && !strings.HasPrefix(text, "#AUDIOSCROBBLER/1") {
- err = errors.New(fmt.Sprintf("Not a scrobbler log file"))
- }
-
- after, found := strings.CutPrefix(text, "#CLIENT/")
- if found {
- client = strings.Split(after, " ")[0]
- }
- }
-
- if err != nil {
- return client, err
- }
- }
- return client, nil
-}
-
-func writeHeader(writer io.Writer) error {
- headers := []string{
- "#AUDIOSCROBBLER/1.1\n",
- "#TZ/UNKNOWN\n",
- "#CLIENT/Rockbox unknown $Revision$\n",
- }
- for _, line := range headers {
- _, err := writer.Write([]byte(line))
- if err != nil {
- return err
- }
- }
- return nil
-}
-
-func rowToListen(row []string, client string) (models.Listen, error) {
- var listen models.Listen
- trackNumber, err := strconv.Atoi(row[3])
- if err != nil {
- return listen, err
- }
-
- duration, err := strconv.Atoi(row[4])
- if err != nil {
- return listen, err
- }
-
- timestamp, err := strconv.Atoi(row[6])
- if err != nil {
- return listen, err
- }
-
- listen = models.Listen{
- Track: models.Track{
- ArtistNames: []string{row[0]},
- ReleaseName: row[1],
- TrackName: row[2],
- TrackNumber: trackNumber,
- Duration: time.Duration(duration * int(time.Second)),
- AdditionalInfo: models.AdditionalInfo{
- "rockbox_rating": row[5],
- "media_player": client,
- },
- },
- ListenedAt: time.Unix(int64(timestamp), 0),
- }
-
- if len(row) > 7 {
- listen.Track.RecordingMbid = models.MBID(row[7])
- }
-
- return listen, nil
-}
diff --git a/backends/scrobblerlog/parser.go b/backends/scrobblerlog/parser.go
new file mode 100644
index 0000000..b93eac1
--- /dev/null
+++ b/backends/scrobblerlog/parser.go
@@ -0,0 +1,217 @@
+/*
+Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package scrobblerlog
+
+import (
+ "bufio"
+ "encoding/csv"
+ "errors"
+ "fmt"
+ "io"
+ "strconv"
+ "strings"
+ "time"
+
+ "go.uploadedlobster.com/scotty/models"
+)
+
+type ScrobblerLog struct {
+ Timezone string
+ Client string
+ Listens []models.Listen
+}
+
+func Parse(data io.Reader, includeSkipped bool) (ScrobblerLog, error) {
+ result := ScrobblerLog{
+ Listens: make([]models.Listen, 0),
+ }
+
+ reader := bufio.NewReader(data)
+ err := readHeader(reader, &result)
+ if err != nil {
+ return result, err
+ }
+
+ tsvReader := csv.NewReader(reader)
+ tsvReader.Comma = '\t'
+ // Row length is often flexible
+ tsvReader.FieldsPerRecord = -1
+
+ for {
+ // A row is:
+ // artistName releaseName trackName trackNumber duration rating timestamp recordingMbid
+ row, err := tsvReader.Read()
+ if err == io.EOF {
+ break
+ } else if err != nil {
+ return result, err
+ }
+
+ // fmt.Printf("row: %v\n", row)
+
+ // We consider only the last field (recording MBID) optional
+ if len(row) < 7 {
+ line, _ := tsvReader.FieldPos(0)
+ return result, errors.New(fmt.Sprintf(
+ "Invalid record in scrobblerlog line %v", line))
+ }
+
+ rating := row[5]
+ if !includeSkipped && rating == "S" {
+ continue
+ }
+
+ client := strings.Split(result.Client, " ")[0]
+ listen, err := rowToListen(row, client)
+ if err != nil {
+ return result, err
+ }
+
+ result.Listens = append(result.Listens, listen)
+ }
+
+ return result, nil
+}
+
+func Write(data io.Writer, log *ScrobblerLog) (lastTimestamp time.Time, err error) {
+ err = writeHeader(data, log)
+ if err != nil {
+ return
+ }
+
+ tsvWriter := csv.NewWriter(data)
+ tsvWriter.Comma = '\t'
+
+ for _, listen := range log.Listens {
+ if listen.ListenedAt.Unix() > lastTimestamp.Unix() {
+ lastTimestamp = listen.ListenedAt
+ }
+
+ // A row is:
+ // artistName releaseName trackName trackNumber duration rating timestamp recordingMbid
+ rating, ok := listen.AdditionalInfo["rockbox_rating"].(string)
+ if !ok || rating == "" {
+ rating = "L"
+ }
+ tsvWriter.Write([]string{
+ listen.ArtistName(),
+ listen.ReleaseName,
+ listen.TrackName,
+ strconv.Itoa(listen.TrackNumber),
+ strconv.Itoa(int(listen.Duration.Seconds())),
+ rating,
+ strconv.Itoa(int(listen.ListenedAt.Unix())),
+ string(listen.RecordingMbid),
+ })
+ }
+
+ tsvWriter.Flush()
+ return
+}
+
+func readHeader(reader *bufio.Reader, log *ScrobblerLog) error {
+ // Skip header
+ for i := 0; i < 3; i++ {
+ line, _, err := reader.ReadLine()
+ if err != nil {
+ return err
+ }
+
+ if len(line) == 0 || line[0] != '#' {
+ err = errors.New(fmt.Sprintf("Unexpected header (line %v)", i))
+ } else {
+ text := string(line)
+ if i == 0 && !strings.HasPrefix(text, "#AUDIOSCROBBLER/1") {
+ err = errors.New(fmt.Sprintf("Not a scrobbler log file"))
+ }
+
+ timezone, found := strings.CutPrefix(text, "#TZ/")
+ if strings.HasPrefix(text, "#TZ/") {
+ log.Timezone = timezone
+ }
+
+ client, found := strings.CutPrefix(text, "#CLIENT/")
+ if found {
+ log.Client = client
+ }
+ }
+
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func writeHeader(writer io.Writer, log *ScrobblerLog) error {
+ headers := []string{
+ "#AUDIOSCROBBLER/1.1\n",
+ "#TZ/" + log.Timezone + "\n",
+ "#CLIENT/" + log.Client + "\n",
+ }
+ for _, line := range headers {
+ _, err := writer.Write([]byte(line))
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func rowToListen(row []string, client string) (models.Listen, error) {
+ var listen models.Listen
+ trackNumber, err := strconv.Atoi(row[3])
+ if err != nil {
+ return listen, err
+ }
+
+ duration, err := strconv.Atoi(row[4])
+ if err != nil {
+ return listen, err
+ }
+
+ timestamp, err := strconv.Atoi(row[6])
+ if err != nil {
+ return listen, err
+ }
+
+ listen = models.Listen{
+ Track: models.Track{
+ ArtistNames: []string{row[0]},
+ ReleaseName: row[1],
+ TrackName: row[2],
+ TrackNumber: trackNumber,
+ Duration: time.Duration(duration * int(time.Second)),
+ AdditionalInfo: models.AdditionalInfo{
+ "rockbox_rating": row[5],
+ "media_player": client,
+ },
+ },
+ ListenedAt: time.Unix(int64(timestamp), 0),
+ }
+
+ if len(row) > 7 {
+ listen.Track.RecordingMbid = models.MBID(row[7])
+ }
+
+ return listen, nil
+}
diff --git a/backends/scrobblerlog/parser_test.go b/backends/scrobblerlog/parser_test.go
new file mode 100644
index 0000000..9cfe480
--- /dev/null
+++ b/backends/scrobblerlog/parser_test.go
@@ -0,0 +1,114 @@
+/*
+Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package scrobblerlog_test
+
+import (
+ "bytes"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "go.uploadedlobster.com/scotty/backends/scrobblerlog"
+ "go.uploadedlobster.com/scotty/models"
+)
+
+var testScrobblerLog = `#AUDIOSCROBBLER/1.1
+#TZ/UNKNOWN
+#CLIENT/Rockbox sansaclipplus $Revision$
+Özcan Deniz Ses ve Ayrilik Sevdanin rengi (sipacik) byMrTurkey 5 306 L 1260342084
+Özcan Deniz Hediye 2@V@7 Bir Dudaktan 1 210 L 1260342633
+KOMPROMAT Traum und Existenz Possession 1 220 L 1260357290 d66b1084-b2ae-4661-8382-5d0c1c484b6d
+Kraftwerk Trans-Europe Express The Hall of Mirrors 2 474 S 1260358000 385ba9e9-626d-4750-a607-58e541dca78e
+Teeth Agency You Don't Have To Live In Pain Wolfs Jam 2 107 L 1260359404 1262beaf-19f8-4534-b9ed-7eef9ca8e83f
+`
+
+func TestParser(t *testing.T) {
+ assert := assert.New(t)
+ data := bytes.NewBufferString(testScrobblerLog)
+ result, err := scrobblerlog.Parse(data, true)
+ require.NoError(t, err)
+ assert.Equal("UNKNOWN", result.Timezone)
+ assert.Equal("Rockbox sansaclipplus $Revision$", result.Client)
+ assert.Len(result.Listens, 5)
+ listen1 := result.Listens[0]
+ assert.Equal("Özcan Deniz", listen1.ArtistName())
+ assert.Equal("Ses ve Ayrilik", listen1.ReleaseName)
+ assert.Equal("Sevdanin rengi (sipacik) byMrTurkey", listen1.TrackName)
+ assert.Equal(5, listen1.TrackNumber)
+ assert.Equal(time.Duration(306*time.Second), listen1.Duration)
+ assert.Equal("L", listen1.AdditionalInfo["rockbox_rating"])
+ assert.Equal(time.Unix(1260342084, 0), listen1.ListenedAt)
+ assert.Equal(models.MBID(""), listen1.RecordingMbid)
+ listen4 := result.Listens[3]
+ assert.Equal("S", listen4.AdditionalInfo["rockbox_rating"])
+ assert.Equal(models.MBID("385ba9e9-626d-4750-a607-58e541dca78e"), listen4.RecordingMbid)
+}
+
+func TestParserExcludeSkipped(t *testing.T) {
+ assert := assert.New(t)
+ data := bytes.NewBufferString(testScrobblerLog)
+ result, err := scrobblerlog.Parse(data, false)
+ require.NoError(t, err)
+ assert.Len(result.Listens, 4)
+ listen4 := result.Listens[3]
+ assert.Equal("L", listen4.AdditionalInfo["rockbox_rating"])
+ assert.Equal(models.MBID("1262beaf-19f8-4534-b9ed-7eef9ca8e83f"), listen4.RecordingMbid)
+}
+
+func TestWrite(t *testing.T) {
+ assert := assert.New(t)
+ data := make([]byte, 0, 10)
+ buffer := bytes.NewBuffer(data)
+ log := scrobblerlog.ScrobblerLog{
+ Timezone: "Unknown",
+ Client: "Rockbox foo $Revision$",
+ Listens: []models.Listen{
+ {
+ ListenedAt: time.Unix(1699572072, 0),
+ Track: models.Track{
+ ArtistNames: []string{"Prinzhorn Dance School"},
+ ReleaseName: "Home Economics",
+ TrackName: "Reign",
+ TrackNumber: 1,
+ Duration: 271 * time.Second,
+ RecordingMbid: models.MBID("b59cf4e7-caee-4019-a844-79d2c58d4dff"),
+ AdditionalInfo: models.AdditionalInfo{"rockbox_rating": "L"},
+ },
+ },
+ },
+ }
+ lastTimestamp, err := scrobblerlog.Write(buffer, &log)
+ require.NoError(t, err)
+ result := string(buffer.Bytes())
+ lines := strings.Split(result, "\n")
+ assert.Equal(5, len(lines))
+ assert.Equal("#AUDIOSCROBBLER/1.1", lines[0])
+ assert.Equal("#TZ/Unknown", lines[1])
+ assert.Equal("#CLIENT/Rockbox foo $Revision$", lines[2])
+ assert.Equal(
+ "Prinzhorn Dance School\tHome Economics\tReign\t1\t271\tL\t1699572072\tb59cf4e7-caee-4019-a844-79d2c58d4dff",
+ lines[3])
+ assert.Equal("", lines[4])
+ assert.Equal(time.Unix(1699572072, 0), lastTimestamp)
+}