Restructured code, moved all modules into internal

For now all modules are considered internal. This might change later

internal/backends/scrobblerlog/parser.go

@@ -0,0 +1,212 @@
+/*
+Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package scrobblerlog
+
+import (
+	"bufio"
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"go.uploadedlobster.com/scotty/internal/models"
+)
+
+type ScrobblerLog struct {
+	Timezone string
+	Client   string
+	Listens  models.ListensList
+}
+
+func Parse(data io.Reader, includeSkipped bool) (ScrobblerLog, error) {
+	result := ScrobblerLog{
+		Listens: make(models.ListensList, 0),
+	}
+
+	reader := bufio.NewReader(data)
+	err := ReadHeader(reader, &result)
+	if err != nil {
+		return result, err
+	}
+
+	tsvReader := csv.NewReader(reader)
+	tsvReader.Comma = '\t'
+	// Row length is often flexible
+	tsvReader.FieldsPerRecord = -1
+
+	for {
+		// A row is:
+		// artistName	releaseName	trackName	trackNumber	duration	rating	timestamp	recordingMbid
+		row, err := tsvReader.Read()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return result, err
+		}
+
+		// fmt.Printf("row: %v\n", row)
+
+		// We consider only the last field (recording MBID) optional
+		if len(row) < 7 {
+			line, _ := tsvReader.FieldPos(0)
+			return result, errors.New(fmt.Sprintf(
+				"Invalid record in scrobblerlog line %v", line))
+		}
+
+		rating := row[5]
+		if !includeSkipped && rating == "S" {
+			continue
+		}
+
+		client := strings.Split(result.Client, " ")[0]
+		listen, err := rowToListen(row, client)
+		if err != nil {
+			return result, err
+		}
+
+		result.Listens = append(result.Listens, listen)
+	}
+
+	return result, nil
+}
+
+func Write(data io.Writer, listens models.ListensList) (lastTimestamp time.Time, err error) {
+	tsvWriter := csv.NewWriter(data)
+	tsvWriter.Comma = '\t'
+
+	for _, listen := range listens {
+		if listen.ListenedAt.Unix() > lastTimestamp.Unix() {
+			lastTimestamp = listen.ListenedAt
+		}
+
+		// A row is:
+		// artistName	releaseName	trackName	trackNumber	duration	rating	timestamp	recordingMbid
+		rating, ok := listen.AdditionalInfo["rockbox_rating"].(string)
+		if !ok || rating == "" {
+			rating = "L"
+		}
+		tsvWriter.Write([]string{
+			listen.ArtistName(),
+			listen.ReleaseName,
+			listen.TrackName,
+			strconv.Itoa(listen.TrackNumber),
+			strconv.Itoa(int(listen.Duration.Seconds())),
+			rating,
+			strconv.Itoa(int(listen.ListenedAt.Unix())),
+			string(listen.RecordingMbid),
+		})
+	}
+
+	tsvWriter.Flush()
+	return
+}
+
+func ReadHeader(reader *bufio.Reader, log *ScrobblerLog) error {
+	// Skip header
+	for i := 0; i < 3; i++ {
+		line, _, err := reader.ReadLine()
+		if err != nil {
+			return err
+		}
+
+		if len(line) == 0 || line[0] != '#' {
+			err = errors.New(fmt.Sprintf("Unexpected header (line %v)", i))
+		} else {
+			text := string(line)
+			if i == 0 && !strings.HasPrefix(text, "#AUDIOSCROBBLER/1") {
+				err = errors.New(fmt.Sprintf("Not a scrobbler log file"))
+			}
+
+			timezone, found := strings.CutPrefix(text, "#TZ/")
+			if strings.HasPrefix(text, "#TZ/") {
+				log.Timezone = timezone
+			}
+
+			client, found := strings.CutPrefix(text, "#CLIENT/")
+			if found {
+				log.Client = client
+			}
+		}
+
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func WriteHeader(writer io.Writer, log *ScrobblerLog) error {
+	headers := []string{
+		"#AUDIOSCROBBLER/1.1\n",
+		"#TZ/" + log.Timezone + "\n",
+		"#CLIENT/" + log.Client + "\n",
+	}
+	for _, line := range headers {
+		_, err := writer.Write([]byte(line))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func rowToListen(row []string, client string) (models.Listen, error) {
+	var listen models.Listen
+	trackNumber, err := strconv.Atoi(row[3])
+	if err != nil {
+		return listen, err
+	}
+
+	duration, err := strconv.Atoi(row[4])
+	if err != nil {
+		return listen, err
+	}
+
+	timestamp, err := strconv.Atoi(row[6])
+	if err != nil {
+		return listen, err
+	}
+
+	listen = models.Listen{
+		Track: models.Track{
+			ArtistNames: []string{row[0]},
+			ReleaseName: row[1],
+			TrackName:   row[2],
+			TrackNumber: trackNumber,
+			Duration:    time.Duration(duration * int(time.Second)),
+			AdditionalInfo: models.AdditionalInfo{
+				"rockbox_rating": row[5],
+				"media_player":   client,
+			},
+		},
+		ListenedAt: time.Unix(int64(timestamp), 0),
+	}
+
+	if len(row) > 7 {
+		listen.Track.RecordingMbid = models.MBID(row[7])
+	}
+
+	return listen, nil
+}

internal/backends/scrobblerlog/parser_test.go

@@ -0,0 +1,128 @@
+/*
+Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package scrobblerlog_test
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uploadedlobster.com/scotty/internal/backends/scrobblerlog"
+	"go.uploadedlobster.com/scotty/internal/models"
+)
+
+var testScrobblerLog = `#AUDIOSCROBBLER/1.1
+#TZ/UNKNOWN
+#CLIENT/Rockbox sansaclipplus $Revision$
+Özcan Deniz	Ses ve Ayrilik	Sevdanin rengi (sipacik) byMrTurkey	5	306	L	1260342084
+Özcan Deniz	Hediye 2@V@7	Bir Dudaktan	1	210	L	1260342633
+KOMPROMAT	Traum und Existenz	Possession	1	220	L	1260357290	d66b1084-b2ae-4661-8382-5d0c1c484b6d
+Kraftwerk	Trans-Europe Express	The Hall of Mirrors	2	474	S	1260358000	385ba9e9-626d-4750-a607-58e541dca78e
+Teeth Agency	You Don't Have To Live In Pain	Wolfs Jam	2	107	L	1260359404	1262beaf-19f8-4534-b9ed-7eef9ca8e83f
+`
+
+func TestParser(t *testing.T) {
+	assert := assert.New(t)
+	data := bytes.NewBufferString(testScrobblerLog)
+	result, err := scrobblerlog.Parse(data, true)
+	require.NoError(t, err)
+	assert.Equal("UNKNOWN", result.Timezone)
+	assert.Equal("Rockbox sansaclipplus $Revision$", result.Client)
+	assert.Len(result.Listens, 5)
+	listen1 := result.Listens[0]
+	assert.Equal("Özcan Deniz", listen1.ArtistName())
+	assert.Equal("Ses ve Ayrilik", listen1.ReleaseName)
+	assert.Equal("Sevdanin rengi (sipacik) byMrTurkey", listen1.TrackName)
+	assert.Equal(5, listen1.TrackNumber)
+	assert.Equal(time.Duration(306*time.Second), listen1.Duration)
+	assert.Equal("L", listen1.AdditionalInfo["rockbox_rating"])
+	assert.Equal(time.Unix(1260342084, 0), listen1.ListenedAt)
+	assert.Equal(models.MBID(""), listen1.RecordingMbid)
+	listen4 := result.Listens[3]
+	assert.Equal("S", listen4.AdditionalInfo["rockbox_rating"])
+	assert.Equal(models.MBID("385ba9e9-626d-4750-a607-58e541dca78e"), listen4.RecordingMbid)
+}
+
+func TestParserExcludeSkipped(t *testing.T) {
+	assert := assert.New(t)
+	data := bytes.NewBufferString(testScrobblerLog)
+	result, err := scrobblerlog.Parse(data, false)
+	require.NoError(t, err)
+	assert.Len(result.Listens, 4)
+	listen4 := result.Listens[3]
+	assert.Equal("L", listen4.AdditionalInfo["rockbox_rating"])
+	assert.Equal(models.MBID("1262beaf-19f8-4534-b9ed-7eef9ca8e83f"), listen4.RecordingMbid)
+}
+
+func TestWrite(t *testing.T) {
+	assert := assert.New(t)
+	data := make([]byte, 0, 10)
+	buffer := bytes.NewBuffer(data)
+	log := scrobblerlog.ScrobblerLog{
+		Timezone: "Unknown",
+		Client:   "Rockbox foo $Revision$",
+		Listens: []models.Listen{
+			{
+				ListenedAt: time.Unix(1699572072, 0),
+				Track: models.Track{
+					ArtistNames:    []string{"Prinzhorn Dance School"},
+					ReleaseName:    "Home Economics",
+					TrackName:      "Reign",
+					TrackNumber:    1,
+					Duration:       271 * time.Second,
+					RecordingMbid:  models.MBID("b59cf4e7-caee-4019-a844-79d2c58d4dff"),
+					AdditionalInfo: models.AdditionalInfo{"rockbox_rating": "L"},
+				},
+			},
+		},
+	}
+	err := scrobblerlog.WriteHeader(buffer, &log)
+	require.NoError(t, err)
+	lastTimestamp, err := scrobblerlog.Write(buffer, log.Listens)
+	require.NoError(t, err)
+	result := string(buffer.Bytes())
+	lines := strings.Split(result, "\n")
+	assert.Equal(5, len(lines))
+	assert.Equal("#AUDIOSCROBBLER/1.1", lines[0])
+	assert.Equal("#TZ/Unknown", lines[1])
+	assert.Equal("#CLIENT/Rockbox foo $Revision$", lines[2])
+	assert.Equal(
+		"Prinzhorn Dance School\tHome Economics\tReign\t1\t271\tL\t1699572072\tb59cf4e7-caee-4019-a844-79d2c58d4dff",
+		lines[3])
+	assert.Equal("", lines[4])
+	assert.Equal(time.Unix(1699572072, 0), lastTimestamp)
+}
+
+func TestReadHeader(t *testing.T) {
+	data := bytes.NewBufferString(testScrobblerLog)
+	reader := bufio.NewReader(data)
+	log := scrobblerlog.ScrobblerLog{}
+	err := scrobblerlog.ReadHeader(reader, &log)
+	assert.NoError(t, err)
+	assert.Equal(t, log.Timezone, "UNKNOWN")
+	assert.Equal(t, log.Client, "Rockbox sansaclipplus $Revision$")
+	assert.Empty(t, log.Listens)
+}

internal/backends/scrobblerlog/scrobblerlog.go

@@ -0,0 +1,136 @@
+/*
+Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+
+This file is part of Scotty.
+
+Scotty is free software: you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later version.
+
+Scotty is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+Scotty. If not, see <https://www.gnu.org/licenses/>.
+*/
+package scrobblerlog
+
+import (
+	"bufio"
+	"os"
+	"sort"
+	"time"
+
+	"github.com/spf13/viper"
+	"go.uploadedlobster.com/scotty/internal/models"
+)
+
+type ScrobblerLogBackend struct {
+	filePath       string
+	includeSkipped bool
+	append         bool
+	file           *os.File
+	log            ScrobblerLog
+}
+
+func (b *ScrobblerLogBackend) Name() string { return "scrobbler-log" }
+
+func (b *ScrobblerLogBackend) FromConfig(config *viper.Viper) models.Backend {
+	b.filePath = config.GetString("file-path")
+	b.includeSkipped = config.GetBool("include-skipped")
+	b.append = true
+	if config.IsSet("append") {
+		b.append = config.GetBool("append")
+	}
+	b.log = ScrobblerLog{
+		Timezone: "UNKNOWN",
+		Client:   "Rockbox unknown $Revision$",
+	}
+	return b
+}
+
+func (b *ScrobblerLogBackend) StartImport() error {
+	flags := os.O_RDWR | os.O_CREATE
+	if !b.append {
+		flags |= os.O_TRUNC
+	}
+	file, err := os.OpenFile(b.filePath, flags, 0666)
+	if err != nil {
+		return err
+	}
+
+	if b.append {
+		stat, err := file.Stat()
+		if err != nil {
+			file.Close()
+			return err
+		}
+
+		if stat.Size() == 0 {
+			// Zero length file, treat as a new file
+			b.append = false
+		} else {
+			// Verify existing file is a scrobbler log
+			reader := bufio.NewReader(file)
+			err = ReadHeader(reader, &b.log)
+			if err != nil {
+				file.Close()
+				return err
+			}
+			file.Seek(0, 2)
+		}
+	}
+
+	if !b.append {
+		err = WriteHeader(file, &b.log)
+		if err != nil {
+			file.Close()
+			return err
+		}
+	}
+
+	b.file = file
+	return nil
+}
+
+func (b *ScrobblerLogBackend) FinishImport() error {
+	return b.file.Close()
+}
+
+func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
+	defer close(results)
+	file, err := os.Open(b.filePath)
+	if err != nil {
+		progress <- models.Progress{}.Complete()
+		results <- models.ListensResult{Error: err}
+		return
+	}
+
+	defer file.Close()
+
+	log, err := Parse(file, b.includeSkipped)
+	if err != nil {
+		progress <- models.Progress{}.Complete()
+		results <- models.ListensResult{Error: err}
+		return
+	}
+
+	listens := log.Listens.NewerThan(oldestTimestamp)
+	sort.Sort(listens)
+	progress <- models.Progress{Elapsed: int64(len(listens))}.Complete()
+	results <- models.ListensResult{Listens: listens}
+}
+
+func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
+	lastTimestamp, err := Write(b.file, export.Listens)
+	if err != nil {
+		return importResult, err
+	}
+
+	importResult.UpdateTimestamp(lastTimestamp)
+	importResult.ImportCount += len(export.Listens)
+	progress <- models.Progress{}.FromImportResult(importResult)
+
+	return importResult, nil
+}

internal/backends/scrobblerlog/scrobblerlog_test.go

@@ -0,0 +1,32 @@
+/*
+Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+
+This file is part of Scotty.
+
+Scotty is free software: you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later version.
+
+Scotty is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+Scotty. If not, see <https://www.gnu.org/licenses/>.
+*/
+package scrobblerlog_test
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"go.uploadedlobster.com/scotty/internal/backends/scrobblerlog"
+)
+
+func TestFromConfig(t *testing.T) {
+	config := viper.New()
+	config.Set("token", "thetoken")
+	backend := (&scrobblerlog.ScrobblerLogBackend{}).FromConfig(config)
+	assert.IsType(t, &scrobblerlog.ScrobblerLogBackend{}, backend)
+}