diff --git a/backends/scrobblerlog.go b/backends/scrobblerlog.go index 01fe8a8..efda58f 100644 --- a/backends/scrobblerlog.go +++ b/backends/scrobblerlog.go @@ -22,17 +22,11 @@ THE SOFTWARE. package backends import ( - "bufio" - "encoding/csv" - "errors" - "fmt" - "io" "os" - "strconv" - "strings" "time" "github.com/spf13/viper" + "go.uploadedlobster.com/scotty/backends/scrobblerlog" "go.uploadedlobster.com/scotty/models" ) @@ -55,51 +49,12 @@ func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time) ([]models. defer file.Close() - reader := bufio.NewReader(file) - client, err := readHeader(reader) + result, err := scrobblerlog.Parse(file, b.includeSkipped) if err != nil { return nil, err } - tsvReader := csv.NewReader(reader) - tsvReader.Comma = '\t' - // Row length is often flexible - tsvReader.FieldsPerRecord = -1 - - listens := make([]models.Listen, 0) - for { - // A row is: - // artistName releaseName trackName trackNumber duration rating timestamp recordingMbid - row, err := tsvReader.Read() - if err == io.EOF { - break - } else if err != nil { - return nil, err - } - - // fmt.Printf("row: %v\n", row) - - // We consider only the last field (recording MBID) optional - if len(row) < 7 { - line, _ := tsvReader.FieldPos(0) - return nil, errors.New(fmt.Sprintf( - "Invalid record in %s line %v", b.filePath, line)) - } - - rating := row[5] - if !b.includeSkipped && rating == "S" { - continue - } - - listen, err := rowToListen(row, client) - if err != nil { - return nil, err - } - - listens = append(listens, listen) - } - - return listens, nil + return result.Listens, nil } func (b ScrobblerLogBackend) ImportListens(listens []models.Listen, oldestTimestamp time.Time) (ImportResult, error) { @@ -115,122 +70,19 @@ func (b ScrobblerLogBackend) ImportListens(listens []models.Listen, oldestTimest defer file.Close() - err = writeHeader(file) + log := scrobblerlog.ScrobblerLog{ + Timezone: "UNKNOWN", + Client: "Rockbox unknown $Revision$", + Listens: listens, + } + + lastTimestamp, err := scrobblerlog.Write(file, &log) + if err != nil { return result, err } - tsvWriter := csv.NewWriter(file) - tsvWriter.Comma = '\t' - - for _, listen := range listens { - result.Count += 1 - if listen.ListenedAt.Unix() > result.LastTimestamp.Unix() { - result.LastTimestamp = listen.ListenedAt - } - - // A row is: - // artistName releaseName trackName trackNumber duration rating timestamp recordingMbid - rating, ok := listen.AdditionalInfo["rockbox_rating"].(string) - if !ok || rating == "" { - rating = "L" - } - tsvWriter.Write([]string{ - listen.ArtistName(), - listen.ReleaseName, - listen.TrackName, - strconv.Itoa(listen.TrackNumber), - strconv.Itoa(int(listen.Duration.Seconds())), - rating, - strconv.Itoa(int(listen.ListenedAt.Unix())), - string(listen.RecordingMbid), - }) - } - - tsvWriter.Flush() - + result.LastTimestamp = lastTimestamp + result.Count = len(listens) return result, nil } - -func readHeader(reader *bufio.Reader) (client string, err error) { - // Skip header - for i := 0; i < 3; i++ { - line, _, err := reader.ReadLine() - if err != nil { - return client, err - } - - if len(line) == 0 || line[0] != '#' { - err = errors.New(fmt.Sprintf("Unexpected header (line %v)", i)) - } else { - text := string(line) - if i == 0 && !strings.HasPrefix(text, "#AUDIOSCROBBLER/1") { - err = errors.New(fmt.Sprintf("Not a scrobbler log file")) - } - - after, found := strings.CutPrefix(text, "#CLIENT/") - if found { - client = strings.Split(after, " ")[0] - } - } - - if err != nil { - return client, err - } - } - return client, nil -} - -func writeHeader(writer io.Writer) error { - headers := []string{ - "#AUDIOSCROBBLER/1.1\n", - "#TZ/UNKNOWN\n", - "#CLIENT/Rockbox unknown $Revision$\n", - } - for _, line := range headers { - _, err := writer.Write([]byte(line)) - if err != nil { - return err - } - } - return nil -} - -func rowToListen(row []string, client string) (models.Listen, error) { - var listen models.Listen - trackNumber, err := strconv.Atoi(row[3]) - if err != nil { - return listen, err - } - - duration, err := strconv.Atoi(row[4]) - if err != nil { - return listen, err - } - - timestamp, err := strconv.Atoi(row[6]) - if err != nil { - return listen, err - } - - listen = models.Listen{ - Track: models.Track{ - ArtistNames: []string{row[0]}, - ReleaseName: row[1], - TrackName: row[2], - TrackNumber: trackNumber, - Duration: time.Duration(duration * int(time.Second)), - AdditionalInfo: models.AdditionalInfo{ - "rockbox_rating": row[5], - "media_player": client, - }, - }, - ListenedAt: time.Unix(int64(timestamp), 0), - } - - if len(row) > 7 { - listen.Track.RecordingMbid = models.MBID(row[7]) - } - - return listen, nil -} diff --git a/backends/scrobblerlog/parser.go b/backends/scrobblerlog/parser.go new file mode 100644 index 0000000..b93eac1 --- /dev/null +++ b/backends/scrobblerlog/parser.go @@ -0,0 +1,217 @@ +/* +Copyright © 2023 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package scrobblerlog + +import ( + "bufio" + "encoding/csv" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" + + "go.uploadedlobster.com/scotty/models" +) + +type ScrobblerLog struct { + Timezone string + Client string + Listens []models.Listen +} + +func Parse(data io.Reader, includeSkipped bool) (ScrobblerLog, error) { + result := ScrobblerLog{ + Listens: make([]models.Listen, 0), + } + + reader := bufio.NewReader(data) + err := readHeader(reader, &result) + if err != nil { + return result, err + } + + tsvReader := csv.NewReader(reader) + tsvReader.Comma = '\t' + // Row length is often flexible + tsvReader.FieldsPerRecord = -1 + + for { + // A row is: + // artistName releaseName trackName trackNumber duration rating timestamp recordingMbid + row, err := tsvReader.Read() + if err == io.EOF { + break + } else if err != nil { + return result, err + } + + // fmt.Printf("row: %v\n", row) + + // We consider only the last field (recording MBID) optional + if len(row) < 7 { + line, _ := tsvReader.FieldPos(0) + return result, errors.New(fmt.Sprintf( + "Invalid record in scrobblerlog line %v", line)) + } + + rating := row[5] + if !includeSkipped && rating == "S" { + continue + } + + client := strings.Split(result.Client, " ")[0] + listen, err := rowToListen(row, client) + if err != nil { + return result, err + } + + result.Listens = append(result.Listens, listen) + } + + return result, nil +} + +func Write(data io.Writer, log *ScrobblerLog) (lastTimestamp time.Time, err error) { + err = writeHeader(data, log) + if err != nil { + return + } + + tsvWriter := csv.NewWriter(data) + tsvWriter.Comma = '\t' + + for _, listen := range log.Listens { + if listen.ListenedAt.Unix() > lastTimestamp.Unix() { + lastTimestamp = listen.ListenedAt + } + + // A row is: + // artistName releaseName trackName trackNumber duration rating timestamp recordingMbid + rating, ok := listen.AdditionalInfo["rockbox_rating"].(string) + if !ok || rating == "" { + rating = "L" + } + tsvWriter.Write([]string{ + listen.ArtistName(), + listen.ReleaseName, + listen.TrackName, + strconv.Itoa(listen.TrackNumber), + strconv.Itoa(int(listen.Duration.Seconds())), + rating, + strconv.Itoa(int(listen.ListenedAt.Unix())), + string(listen.RecordingMbid), + }) + } + + tsvWriter.Flush() + return +} + +func readHeader(reader *bufio.Reader, log *ScrobblerLog) error { + // Skip header + for i := 0; i < 3; i++ { + line, _, err := reader.ReadLine() + if err != nil { + return err + } + + if len(line) == 0 || line[0] != '#' { + err = errors.New(fmt.Sprintf("Unexpected header (line %v)", i)) + } else { + text := string(line) + if i == 0 && !strings.HasPrefix(text, "#AUDIOSCROBBLER/1") { + err = errors.New(fmt.Sprintf("Not a scrobbler log file")) + } + + timezone, found := strings.CutPrefix(text, "#TZ/") + if strings.HasPrefix(text, "#TZ/") { + log.Timezone = timezone + } + + client, found := strings.CutPrefix(text, "#CLIENT/") + if found { + log.Client = client + } + } + + if err != nil { + return err + } + } + return nil +} + +func writeHeader(writer io.Writer, log *ScrobblerLog) error { + headers := []string{ + "#AUDIOSCROBBLER/1.1\n", + "#TZ/" + log.Timezone + "\n", + "#CLIENT/" + log.Client + "\n", + } + for _, line := range headers { + _, err := writer.Write([]byte(line)) + if err != nil { + return err + } + } + return nil +} + +func rowToListen(row []string, client string) (models.Listen, error) { + var listen models.Listen + trackNumber, err := strconv.Atoi(row[3]) + if err != nil { + return listen, err + } + + duration, err := strconv.Atoi(row[4]) + if err != nil { + return listen, err + } + + timestamp, err := strconv.Atoi(row[6]) + if err != nil { + return listen, err + } + + listen = models.Listen{ + Track: models.Track{ + ArtistNames: []string{row[0]}, + ReleaseName: row[1], + TrackName: row[2], + TrackNumber: trackNumber, + Duration: time.Duration(duration * int(time.Second)), + AdditionalInfo: models.AdditionalInfo{ + "rockbox_rating": row[5], + "media_player": client, + }, + }, + ListenedAt: time.Unix(int64(timestamp), 0), + } + + if len(row) > 7 { + listen.Track.RecordingMbid = models.MBID(row[7]) + } + + return listen, nil +} diff --git a/backends/scrobblerlog/parser_test.go b/backends/scrobblerlog/parser_test.go new file mode 100644 index 0000000..9cfe480 --- /dev/null +++ b/backends/scrobblerlog/parser_test.go @@ -0,0 +1,114 @@ +/* +Copyright © 2023 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package scrobblerlog_test + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uploadedlobster.com/scotty/backends/scrobblerlog" + "go.uploadedlobster.com/scotty/models" +) + +var testScrobblerLog = `#AUDIOSCROBBLER/1.1 +#TZ/UNKNOWN +#CLIENT/Rockbox sansaclipplus $Revision$ +Özcan Deniz Ses ve Ayrilik Sevdanin rengi (sipacik) byMrTurkey 5 306 L 1260342084 +Özcan Deniz Hediye 2@V@7 Bir Dudaktan 1 210 L 1260342633 +KOMPROMAT Traum und Existenz Possession 1 220 L 1260357290 d66b1084-b2ae-4661-8382-5d0c1c484b6d +Kraftwerk Trans-Europe Express The Hall of Mirrors 2 474 S 1260358000 385ba9e9-626d-4750-a607-58e541dca78e +Teeth Agency You Don't Have To Live In Pain Wolfs Jam 2 107 L 1260359404 1262beaf-19f8-4534-b9ed-7eef9ca8e83f +` + +func TestParser(t *testing.T) { + assert := assert.New(t) + data := bytes.NewBufferString(testScrobblerLog) + result, err := scrobblerlog.Parse(data, true) + require.NoError(t, err) + assert.Equal("UNKNOWN", result.Timezone) + assert.Equal("Rockbox sansaclipplus $Revision$", result.Client) + assert.Len(result.Listens, 5) + listen1 := result.Listens[0] + assert.Equal("Özcan Deniz", listen1.ArtistName()) + assert.Equal("Ses ve Ayrilik", listen1.ReleaseName) + assert.Equal("Sevdanin rengi (sipacik) byMrTurkey", listen1.TrackName) + assert.Equal(5, listen1.TrackNumber) + assert.Equal(time.Duration(306*time.Second), listen1.Duration) + assert.Equal("L", listen1.AdditionalInfo["rockbox_rating"]) + assert.Equal(time.Unix(1260342084, 0), listen1.ListenedAt) + assert.Equal(models.MBID(""), listen1.RecordingMbid) + listen4 := result.Listens[3] + assert.Equal("S", listen4.AdditionalInfo["rockbox_rating"]) + assert.Equal(models.MBID("385ba9e9-626d-4750-a607-58e541dca78e"), listen4.RecordingMbid) +} + +func TestParserExcludeSkipped(t *testing.T) { + assert := assert.New(t) + data := bytes.NewBufferString(testScrobblerLog) + result, err := scrobblerlog.Parse(data, false) + require.NoError(t, err) + assert.Len(result.Listens, 4) + listen4 := result.Listens[3] + assert.Equal("L", listen4.AdditionalInfo["rockbox_rating"]) + assert.Equal(models.MBID("1262beaf-19f8-4534-b9ed-7eef9ca8e83f"), listen4.RecordingMbid) +} + +func TestWrite(t *testing.T) { + assert := assert.New(t) + data := make([]byte, 0, 10) + buffer := bytes.NewBuffer(data) + log := scrobblerlog.ScrobblerLog{ + Timezone: "Unknown", + Client: "Rockbox foo $Revision$", + Listens: []models.Listen{ + { + ListenedAt: time.Unix(1699572072, 0), + Track: models.Track{ + ArtistNames: []string{"Prinzhorn Dance School"}, + ReleaseName: "Home Economics", + TrackName: "Reign", + TrackNumber: 1, + Duration: 271 * time.Second, + RecordingMbid: models.MBID("b59cf4e7-caee-4019-a844-79d2c58d4dff"), + AdditionalInfo: models.AdditionalInfo{"rockbox_rating": "L"}, + }, + }, + }, + } + lastTimestamp, err := scrobblerlog.Write(buffer, &log) + require.NoError(t, err) + result := string(buffer.Bytes()) + lines := strings.Split(result, "\n") + assert.Equal(5, len(lines)) + assert.Equal("#AUDIOSCROBBLER/1.1", lines[0]) + assert.Equal("#TZ/Unknown", lines[1]) + assert.Equal("#CLIENT/Rockbox foo $Revision$", lines[2]) + assert.Equal( + "Prinzhorn Dance School\tHome Economics\tReign\t1\t271\tL\t1699572072\tb59cf4e7-caee-4019-a844-79d2c58d4dff", + lines[3]) + assert.Equal("", lines[4]) + assert.Equal(time.Unix(1699572072, 0), lastTimestamp) +}