mirror of
https://git.sr.ht/~phw/scotty
synced 2025-04-16 10:09:28 +02:00
Tests for reading / writing scrobbler log
This commit is contained in:
Philipp Wolfer
parent
9d97e324aa
commit
cf8a6d2ab6
3 changed files with 344 additions and 161 deletions
|
|
@ -22,17 +22,11 @@ THE SOFTWARE.
|
|||
package backends
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
"go.uploadedlobster.com/scotty/backends/scrobblerlog"
|
||||
"go.uploadedlobster.com/scotty/models"
|
||||
)
|
||||
|
||||
|
|
@ -55,51 +49,12 @@ func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time) ([]models.
|
|||
|
||||
defer file.Close()
|
||||
|
||||
reader := bufio.NewReader(file)
|
||||
client, err := readHeader(reader)
|
||||
result, err := scrobblerlog.Parse(file, b.includeSkipped)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tsvReader := csv.NewReader(reader)
|
||||
tsvReader.Comma = '\t'
|
||||
// Row length is often flexible
|
||||
tsvReader.FieldsPerRecord = -1
|
||||
|
||||
listens := make([]models.Listen, 0)
|
||||
for {
|
||||
// A row is:
|
||||
// artistName releaseName trackName trackNumber duration rating timestamp recordingMbid
|
||||
row, err := tsvReader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// fmt.Printf("row: %v\n", row)
|
||||
|
||||
// We consider only the last field (recording MBID) optional
|
||||
if len(row) < 7 {
|
||||
line, _ := tsvReader.FieldPos(0)
|
||||
return nil, errors.New(fmt.Sprintf(
|
||||
"Invalid record in %s line %v", b.filePath, line))
|
||||
}
|
||||
|
||||
rating := row[5]
|
||||
if !b.includeSkipped && rating == "S" {
|
||||
continue
|
||||
}
|
||||
|
||||
listen, err := rowToListen(row, client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listens = append(listens, listen)
|
||||
}
|
||||
|
||||
return listens, nil
|
||||
return result.Listens, nil
|
||||
}
|
||||
|
||||
func (b ScrobblerLogBackend) ImportListens(listens []models.Listen, oldestTimestamp time.Time) (ImportResult, error) {
|
||||
|
|
@ -115,122 +70,19 @@ func (b ScrobblerLogBackend) ImportListens(listens []models.Listen, oldestTimest
|
|||
|
||||
defer file.Close()
|
||||
|
||||
err = writeHeader(file)
|
||||
log := scrobblerlog.ScrobblerLog{
|
||||
Timezone: "UNKNOWN",
|
||||
Client: "Rockbox unknown $Revision$",
|
||||
Listens: listens,
|
||||
}
|
||||
|
||||
lastTimestamp, err := scrobblerlog.Write(file, &log)
|
||||
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
tsvWriter := csv.NewWriter(file)
|
||||
tsvWriter.Comma = '\t'
|
||||
|
||||
for _, listen := range listens {
|
||||
result.Count += 1
|
||||
if listen.ListenedAt.Unix() > result.LastTimestamp.Unix() {
|
||||
result.LastTimestamp = listen.ListenedAt
|
||||
}
|
||||
|
||||
// A row is:
|
||||
// artistName releaseName trackName trackNumber duration rating timestamp recordingMbid
|
||||
rating, ok := listen.AdditionalInfo["rockbox_rating"].(string)
|
||||
if !ok || rating == "" {
|
||||
rating = "L"
|
||||
}
|
||||
tsvWriter.Write([]string{
|
||||
listen.ArtistName(),
|
||||
listen.ReleaseName,
|
||||
listen.TrackName,
|
||||
strconv.Itoa(listen.TrackNumber),
|
||||
strconv.Itoa(int(listen.Duration.Seconds())),
|
||||
rating,
|
||||
strconv.Itoa(int(listen.ListenedAt.Unix())),
|
||||
string(listen.RecordingMbid),
|
||||
})
|
||||
}
|
||||
|
||||
tsvWriter.Flush()
|
||||
|
||||
result.LastTimestamp = lastTimestamp
|
||||
result.Count = len(listens)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func readHeader(reader *bufio.Reader) (client string, err error) {
|
||||
// Skip header
|
||||
for i := 0; i < 3; i++ {
|
||||
line, _, err := reader.ReadLine()
|
||||
if err != nil {
|
||||
return client, err
|
||||
}
|
||||
|
||||
if len(line) == 0 || line[0] != '#' {
|
||||
err = errors.New(fmt.Sprintf("Unexpected header (line %v)", i))
|
||||
} else {
|
||||
text := string(line)
|
||||
if i == 0 && !strings.HasPrefix(text, "#AUDIOSCROBBLER/1") {
|
||||
err = errors.New(fmt.Sprintf("Not a scrobbler log file"))
|
||||
}
|
||||
|
||||
after, found := strings.CutPrefix(text, "#CLIENT/")
|
||||
if found {
|
||||
client = strings.Split(after, " ")[0]
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return client, err
|
||||
}
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func writeHeader(writer io.Writer) error {
|
||||
headers := []string{
|
||||
"#AUDIOSCROBBLER/1.1\n",
|
||||
"#TZ/UNKNOWN\n",
|
||||
"#CLIENT/Rockbox unknown $Revision$\n",
|
||||
}
|
||||
for _, line := range headers {
|
||||
_, err := writer.Write([]byte(line))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rowToListen(row []string, client string) (models.Listen, error) {
|
||||
var listen models.Listen
|
||||
trackNumber, err := strconv.Atoi(row[3])
|
||||
if err != nil {
|
||||
return listen, err
|
||||
}
|
||||
|
||||
duration, err := strconv.Atoi(row[4])
|
||||
if err != nil {
|
||||
return listen, err
|
||||
}
|
||||
|
||||
timestamp, err := strconv.Atoi(row[6])
|
||||
if err != nil {
|
||||
return listen, err
|
||||
}
|
||||
|
||||
listen = models.Listen{
|
||||
Track: models.Track{
|
||||
ArtistNames: []string{row[0]},
|
||||
ReleaseName: row[1],
|
||||
TrackName: row[2],
|
||||
TrackNumber: trackNumber,
|
||||
Duration: time.Duration(duration * int(time.Second)),
|
||||
AdditionalInfo: models.AdditionalInfo{
|
||||
"rockbox_rating": row[5],
|
||||
"media_player": client,
|
||||
},
|
||||
},
|
||||
ListenedAt: time.Unix(int64(timestamp), 0),
|
||||
}
|
||||
|
||||
if len(row) > 7 {
|
||||
listen.Track.RecordingMbid = models.MBID(row[7])
|
||||
}
|
||||
|
||||
return listen, nil
|
||||
}
|
||||
|
|
|
|||
217
backends/scrobblerlog/parser.go
Normal file
217
backends/scrobblerlog/parser.go
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
/*
|
||||
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
package scrobblerlog
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uploadedlobster.com/scotty/models"
|
||||
)
|
||||
|
||||
type ScrobblerLog struct {
|
||||
Timezone string
|
||||
Client string
|
||||
Listens []models.Listen
|
||||
}
|
||||
|
||||
func Parse(data io.Reader, includeSkipped bool) (ScrobblerLog, error) {
|
||||
result := ScrobblerLog{
|
||||
Listens: make([]models.Listen, 0),
|
||||
}
|
||||
|
||||
reader := bufio.NewReader(data)
|
||||
err := readHeader(reader, &result)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
tsvReader := csv.NewReader(reader)
|
||||
tsvReader.Comma = '\t'
|
||||
// Row length is often flexible
|
||||
tsvReader.FieldsPerRecord = -1
|
||||
|
||||
for {
|
||||
// A row is:
|
||||
// artistName releaseName trackName trackNumber duration rating timestamp recordingMbid
|
||||
row, err := tsvReader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
// fmt.Printf("row: %v\n", row)
|
||||
|
||||
// We consider only the last field (recording MBID) optional
|
||||
if len(row) < 7 {
|
||||
line, _ := tsvReader.FieldPos(0)
|
||||
return result, errors.New(fmt.Sprintf(
|
||||
"Invalid record in scrobblerlog line %v", line))
|
||||
}
|
||||
|
||||
rating := row[5]
|
||||
if !includeSkipped && rating == "S" {
|
||||
continue
|
||||
}
|
||||
|
||||
client := strings.Split(result.Client, " ")[0]
|
||||
listen, err := rowToListen(row, client)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
result.Listens = append(result.Listens, listen)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func Write(data io.Writer, log *ScrobblerLog) (lastTimestamp time.Time, err error) {
|
||||
err = writeHeader(data, log)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
tsvWriter := csv.NewWriter(data)
|
||||
tsvWriter.Comma = '\t'
|
||||
|
||||
for _, listen := range log.Listens {
|
||||
if listen.ListenedAt.Unix() > lastTimestamp.Unix() {
|
||||
lastTimestamp = listen.ListenedAt
|
||||
}
|
||||
|
||||
// A row is:
|
||||
// artistName releaseName trackName trackNumber duration rating timestamp recordingMbid
|
||||
rating, ok := listen.AdditionalInfo["rockbox_rating"].(string)
|
||||
if !ok || rating == "" {
|
||||
rating = "L"
|
||||
}
|
||||
tsvWriter.Write([]string{
|
||||
listen.ArtistName(),
|
||||
listen.ReleaseName,
|
||||
listen.TrackName,
|
||||
strconv.Itoa(listen.TrackNumber),
|
||||
strconv.Itoa(int(listen.Duration.Seconds())),
|
||||
rating,
|
||||
strconv.Itoa(int(listen.ListenedAt.Unix())),
|
||||
string(listen.RecordingMbid),
|
||||
})
|
||||
}
|
||||
|
||||
tsvWriter.Flush()
|
||||
return
|
||||
}
|
||||
|
||||
func readHeader(reader *bufio.Reader, log *ScrobblerLog) error {
|
||||
// Skip header
|
||||
for i := 0; i < 3; i++ {
|
||||
line, _, err := reader.ReadLine()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(line) == 0 || line[0] != '#' {
|
||||
err = errors.New(fmt.Sprintf("Unexpected header (line %v)", i))
|
||||
} else {
|
||||
text := string(line)
|
||||
if i == 0 && !strings.HasPrefix(text, "#AUDIOSCROBBLER/1") {
|
||||
err = errors.New(fmt.Sprintf("Not a scrobbler log file"))
|
||||
}
|
||||
|
||||
timezone, found := strings.CutPrefix(text, "#TZ/")
|
||||
if strings.HasPrefix(text, "#TZ/") {
|
||||
log.Timezone = timezone
|
||||
}
|
||||
|
||||
client, found := strings.CutPrefix(text, "#CLIENT/")
|
||||
if found {
|
||||
log.Client = client
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeHeader(writer io.Writer, log *ScrobblerLog) error {
|
||||
headers := []string{
|
||||
"#AUDIOSCROBBLER/1.1\n",
|
||||
"#TZ/" + log.Timezone + "\n",
|
||||
"#CLIENT/" + log.Client + "\n",
|
||||
}
|
||||
for _, line := range headers {
|
||||
_, err := writer.Write([]byte(line))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rowToListen(row []string, client string) (models.Listen, error) {
|
||||
var listen models.Listen
|
||||
trackNumber, err := strconv.Atoi(row[3])
|
||||
if err != nil {
|
||||
return listen, err
|
||||
}
|
||||
|
||||
duration, err := strconv.Atoi(row[4])
|
||||
if err != nil {
|
||||
return listen, err
|
||||
}
|
||||
|
||||
timestamp, err := strconv.Atoi(row[6])
|
||||
if err != nil {
|
||||
return listen, err
|
||||
}
|
||||
|
||||
listen = models.Listen{
|
||||
Track: models.Track{
|
||||
ArtistNames: []string{row[0]},
|
||||
ReleaseName: row[1],
|
||||
TrackName: row[2],
|
||||
TrackNumber: trackNumber,
|
||||
Duration: time.Duration(duration * int(time.Second)),
|
||||
AdditionalInfo: models.AdditionalInfo{
|
||||
"rockbox_rating": row[5],
|
||||
"media_player": client,
|
||||
},
|
||||
},
|
||||
ListenedAt: time.Unix(int64(timestamp), 0),
|
||||
}
|
||||
|
||||
if len(row) > 7 {
|
||||
listen.Track.RecordingMbid = models.MBID(row[7])
|
||||
}
|
||||
|
||||
return listen, nil
|
||||
}
|
||||
114
backends/scrobblerlog/parser_test.go
Normal file
114
backends/scrobblerlog/parser_test.go
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
/*
|
||||
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
package scrobblerlog_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uploadedlobster.com/scotty/backends/scrobblerlog"
|
||||
"go.uploadedlobster.com/scotty/models"
|
||||
)
|
||||
|
||||
var testScrobblerLog = `#AUDIOSCROBBLER/1.1
|
||||
#TZ/UNKNOWN
|
||||
#CLIENT/Rockbox sansaclipplus $Revision$
|
||||
Özcan Deniz Ses ve Ayrilik Sevdanin rengi (sipacik) byMrTurkey 5 306 L 1260342084
|
||||
Özcan Deniz Hediye 2@V@7 Bir Dudaktan 1 210 L 1260342633
|
||||
KOMPROMAT Traum und Existenz Possession 1 220 L 1260357290 d66b1084-b2ae-4661-8382-5d0c1c484b6d
|
||||
Kraftwerk Trans-Europe Express The Hall of Mirrors 2 474 S 1260358000 385ba9e9-626d-4750-a607-58e541dca78e
|
||||
Teeth Agency You Don't Have To Live In Pain Wolfs Jam 2 107 L 1260359404 1262beaf-19f8-4534-b9ed-7eef9ca8e83f
|
||||
`
|
||||
|
||||
func TestParser(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
data := bytes.NewBufferString(testScrobblerLog)
|
||||
result, err := scrobblerlog.Parse(data, true)
|
||||
require.NoError(t, err)
|
||||
assert.Equal("UNKNOWN", result.Timezone)
|
||||
assert.Equal("Rockbox sansaclipplus $Revision$", result.Client)
|
||||
assert.Len(result.Listens, 5)
|
||||
listen1 := result.Listens[0]
|
||||
assert.Equal("Özcan Deniz", listen1.ArtistName())
|
||||
assert.Equal("Ses ve Ayrilik", listen1.ReleaseName)
|
||||
assert.Equal("Sevdanin rengi (sipacik) byMrTurkey", listen1.TrackName)
|
||||
assert.Equal(5, listen1.TrackNumber)
|
||||
assert.Equal(time.Duration(306*time.Second), listen1.Duration)
|
||||
assert.Equal("L", listen1.AdditionalInfo["rockbox_rating"])
|
||||
assert.Equal(time.Unix(1260342084, 0), listen1.ListenedAt)
|
||||
assert.Equal(models.MBID(""), listen1.RecordingMbid)
|
||||
listen4 := result.Listens[3]
|
||||
assert.Equal("S", listen4.AdditionalInfo["rockbox_rating"])
|
||||
assert.Equal(models.MBID("385ba9e9-626d-4750-a607-58e541dca78e"), listen4.RecordingMbid)
|
||||
}
|
||||
|
||||
func TestParserExcludeSkipped(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
data := bytes.NewBufferString(testScrobblerLog)
|
||||
result, err := scrobblerlog.Parse(data, false)
|
||||
require.NoError(t, err)
|
||||
assert.Len(result.Listens, 4)
|
||||
listen4 := result.Listens[3]
|
||||
assert.Equal("L", listen4.AdditionalInfo["rockbox_rating"])
|
||||
assert.Equal(models.MBID("1262beaf-19f8-4534-b9ed-7eef9ca8e83f"), listen4.RecordingMbid)
|
||||
}
|
||||
|
||||
func TestWrite(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
data := make([]byte, 0, 10)
|
||||
buffer := bytes.NewBuffer(data)
|
||||
log := scrobblerlog.ScrobblerLog{
|
||||
Timezone: "Unknown",
|
||||
Client: "Rockbox foo $Revision$",
|
||||
Listens: []models.Listen{
|
||||
{
|
||||
ListenedAt: time.Unix(1699572072, 0),
|
||||
Track: models.Track{
|
||||
ArtistNames: []string{"Prinzhorn Dance School"},
|
||||
ReleaseName: "Home Economics",
|
||||
TrackName: "Reign",
|
||||
TrackNumber: 1,
|
||||
Duration: 271 * time.Second,
|
||||
RecordingMbid: models.MBID("b59cf4e7-caee-4019-a844-79d2c58d4dff"),
|
||||
AdditionalInfo: models.AdditionalInfo{"rockbox_rating": "L"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
lastTimestamp, err := scrobblerlog.Write(buffer, &log)
|
||||
require.NoError(t, err)
|
||||
result := string(buffer.Bytes())
|
||||
lines := strings.Split(result, "\n")
|
||||
assert.Equal(5, len(lines))
|
||||
assert.Equal("#AUDIOSCROBBLER/1.1", lines[0])
|
||||
assert.Equal("#TZ/Unknown", lines[1])
|
||||
assert.Equal("#CLIENT/Rockbox foo $Revision$", lines[2])
|
||||
assert.Equal(
|
||||
"Prinzhorn Dance School\tHome Economics\tReign\t1\t271\tL\t1699572072\tb59cf4e7-caee-4019-a844-79d2c58d4dff",
|
||||
lines[3])
|
||||
assert.Equal("", lines[4])
|
||||
assert.Equal(time.Unix(1699572072, 0), lastTimestamp)
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue