marc/scotty
1
0
Fork 0
mirror of https://git.sr.ht/~phw/scotty synced 2025-04-16 10:09:28 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Updated all import/export interfaces

Browse source
This commit is contained in:
Philipp Wolfer 2023-11-15 19:24:12 +01:00
parent 729a3d0ed0
commit ab04eb1123
No known key found for this signature in database
GPG key ID: 8FDF744D4919943B
12 changed files with 247 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
backends/dump/dump.go
View file

@ -37,8 +37,6 @@ func (b DumpBackend) FromConfig(config *viper.Viper) models.Backend {
func (b DumpBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
TotalCount: 0,
ImportCount: 0,
LastTimestamp: oldestTimestamp,
}
for result := range results {
@ -59,8 +57,6 @@ func (b DumpBackend) ImportListens(results chan models.ListensResult, oldestTime
func (b DumpBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
TotalCount: 0,
ImportCount: 0,
LastTimestamp: oldestTimestamp,
}
for result := range results {

29
backends/funkwhale/funkwhale.go
View file

@ -22,7 +22,7 @@ THE SOFTWARE.
package funkwhale
import (
"slices"
"sort"
"time"
"github.com/spf13/viper"
@ -45,17 +45,19 @@ func (b FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend {
return b
}
func (b FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time) ([]models.Listen, error) {
func (b FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
page := 1
perPage := MaxItemsPerGet
listens := make([]models.Listen, 0, 2*MaxItemsPerGet)
// We need to gather the full list of listens in order to sort them
listens := make(models.ListensList, 0, 2*MaxItemsPerGet)
out:
for {
result, err := b.client.GetHistoryListenings(b.username, page, perPage)
if err != nil {
return nil, err
results <- models.ListensResult{Error: err}
close(results)
}
count := len(result.Results)
@ -80,21 +82,26 @@ out:
page += 1
}
slices.Reverse(listens)
return listens, nil
sort.Sort(listens)
results <- models.ListensResult{Listens: listens}
close(results)
}
func (b FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time) ([]models.Love, error) {
func (b FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
page := 1
perPage := MaxItemsPerGet
loves := make([]models.Love, 0, 2*MaxItemsPerGet)
defer close(results)
// We need to gather the full list of listens in order to sort them
loves := make(models.LovesList, 0, 2*MaxItemsPerGet)
out:
for {
result, err := b.client.GetFavoriteTracks(page, perPage)
if err != nil {
return nil, err
results <- models.LovesResult{Error: err}
return
}
count := len(result.Results)
@ -119,8 +126,8 @@ out:
page += 1
}
slices.Reverse(loves)
return loves, nil
sort.Sort(loves)
results <- models.LovesResult{Loves: loves}
}
func (l Listening) ToListen() models.Listen {

41
backends/jspf/jspf.go
View file

@ -46,15 +46,35 @@ func (b JspfBackend) FromConfig(config *viper.Viper) models.Backend {
return b
}
func (b JspfBackend) ImportLoves(loves []models.Love, oldestTimestamp time.Time) (models.ImportResult, error) {
result := models.ImportResult{
TotalCount: len(loves),
ImportCount: 0,
func (b JspfBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
}
tracks := make([]Track, 0, result.TotalCount)
for _, love := range loves {
tracks := make([]Track, 0, importResult.TotalCount)
for result := range results {
if result.Error != nil {
return importResult, result.Error
}
importResult.TotalCount += len(result.Loves)
for _, love := range result.Loves {
track := loveToTrack(love)
tracks = append(tracks, track)
oldestTimestamp = love.Created
importResult.ImportCount += 1
}
}
err := b.writeJspf(tracks)
if err != nil {
importResult.UpdateTimestamp(oldestTimestamp)
importResult.ImportCount = len(tracks)
}
return importResult, err
}
func loveToTrack(love models.Love) Track {
extension := MusicBrainzTrackExtension{
AddedAt: love.Created,
AddedBy: love.UserName,
@ -84,14 +104,7 @@ func (b JspfBackend) ImportLoves(loves []models.Love, oldestTimestamp time.Time)
track.Identifier = append(track.Identifier, "https://musicbrainz.org/recording/"+string(love.RecordingMbid))
}
tracks = append(tracks, track)
result.UpdateTimestamp(love.Created)
result.ImportCount += 1
}
err := b.writeJspf(tracks)
return result, err
return track
}
func (b JspfBackend) writeJspf(tracks []Track) error {

67
backends/listenbrainz/listenbrainz.go
View file

@ -23,7 +23,7 @@ package listenbrainz
import (
"fmt"
"slices"
"sort"
"time"
"github.com/spf13/viper"
@ -42,16 +42,21 @@ func (b ListenBrainzApiBackend) FromConfig(config *viper.Viper) models.Backend {
return b
}
func (b ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time) ([]models.Listen, error) {
func (b ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
maxTime := time.Now()
minTime := time.Unix(0, 0)
listens := make([]models.Listen, 0, 2*MaxItemsPerGet)
defer close(results)
// FIXME: Optimize by fetching the listens in reverse listen time order
listens := make(models.ListensList, 0, 2*MaxItemsPerGet)
out:
for {
result, err := b.client.GetListens(b.username, maxTime, minTime)
if err != nil {
return nil, err
results <- models.ListensResult{Error: err}
return
}
count := len(result.Payload.Listens)
@ -73,19 +78,21 @@ out:
}
}
slices.Reverse(listens)
return listens, nil
sort.Sort(listens)
results <- models.ListensResult{Listens: listens}
}
func (b ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time) ([]models.Love, error) {
func (b ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
offset := 0
loves := make([]models.Love, 0, 2*MaxItemsPerGet)
defer close(results)
loves := make(models.LovesList, 0, 2*MaxItemsPerGet)
out:
for {
result, err := b.client.GetFeedback(b.username, 1, offset)
if err != nil {
return nil, err
results <- models.LovesResult{Error: err}
return
}
count := len(result.Feedback)
@ -105,29 +112,37 @@ out:
offset += MaxItemsPerGet
}
slices.Reverse(loves)
return loves, nil
sort.Sort(loves)
results <- models.LovesResult{Loves: loves}
}
func (b ListenBrainzApiBackend) ImportLoves(loves []models.Love, oldestTimestamp time.Time) (models.ImportResult, error) {
result := models.ImportResult{
TotalCount: len(loves),
ImportCount: 0,
func (b ListenBrainzApiBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
ImportErrors: make([]string, 0),
}
existingLoves, err := b.ExportLoves(time.Unix(0, 0))
if err != nil {
return result, err
existingLovesChan := make(chan models.LovesResult)
go b.ExportLoves(time.Unix(0, 0), existingLovesChan)
existingLoves := <-existingLovesChan
if existingLoves.Error != nil {
results <- models.LovesResult{Error: existingLoves.Error}
close(results)
}
existingMbids := make(map[string]bool, len(existingLoves))
for _, love := range existingLoves {
existingMbids := make(map[string]bool, len(existingLoves.Loves))
for _, love := range existingLoves.Loves {
existingMbids[string(love.RecordingMbid)] = true
}
for _, love := range loves {
for result := range results {
if result.Error != nil {
return importResult, result.Error
}
importResult.TotalCount += len(result.Loves)
for _, love := range result.Loves {
if love.Created.Unix() <= oldestTimestamp.Unix() {
continue
}
@ -158,16 +173,18 @@ func (b ListenBrainzApiBackend) ImportLoves(loves []models.Love, oldestTimestamp
}
if ok {
result.UpdateTimestamp(love.Created)
result.ImportCount += 1
importResult.UpdateTimestamp(love.Created)
importResult.ImportCount += 1
} else {
msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v",
love.TrackName, love.ArtistName(), errMsg)
result.ImportErrors = append(result.ImportErrors, msg)
importResult.ImportErrors = append(importResult.ImportErrors, msg)
}
}
}
return result, nil
}
return importResult, nil
}
func (lbListen Listen) ToListen() models.Listen {

2
backends/maloja/client.go
View file

@ -28,6 +28,8 @@ import (
"github.com/go-resty/resty/v2"
)
const MaxItemsPerGet = 1000
type Client struct {
HttpClient *resty.Client
token string

47
backends/maloja/maloja.go
View file

@ -23,7 +23,7 @@ package maloja
import (
"errors"
"slices"
"sort"
"strings"
"time"
@ -45,17 +45,21 @@ func (b MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend {
return b
}
func (b MalojaApiBackend) ExportListens(oldestTimestamp time.Time) ([]models.Listen, error) {
func (b MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
page := 0
perPage := 1000
perPage := MaxItemsPerGet
listens := make([]models.Listen, 0)
defer close(results)
// We need to gather the full list of listens in order to sort them
listens := make(models.ListensList, 0, 2*perPage)
out:
for {
result, err := b.client.GetScrobbles(page, perPage)
if err != nil {
return nil, err
results <- models.ListensResult{Error: err}
return
}
count := len(result.List)
@ -74,19 +78,24 @@ out:
page += 1
}
slices.Reverse(listens)
return listens, nil
sort.Sort(listens)
results <- models.ListensResult{Listens: listens}
}
func (b MalojaApiBackend) ImportListens(listens []models.Listen, oldestTimestamp time.Time) (models.ImportResult, error) {
result := models.ImportResult{
TotalCount: len(listens),
ImportCount: 0,
func (b MalojaApiBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
}
for _, listen := range listens {
for result := range results {
if result.Error != nil {
return importResult, result.Error
}
importResult.TotalCount += len(result.Listens)
for _, listen := range result.Listens {
if listen.ListenedAt.Unix() <= oldestTimestamp.Unix() {
continue
break
}
scrobble := NewScrobble{
@ -101,17 +110,17 @@ func (b MalojaApiBackend) ImportListens(listens []models.Listen, oldestTimestamp
resp, err := b.client.NewScrobble(scrobble)
if err != nil {
return result, err
return importResult, err
} else if resp.Status != "success" {
return result, errors.New(resp.Error.Description)
return importResult, errors.New(resp.Error.Description)
}
if listen.ListenedAt.Unix() > result.LastTimestamp.Unix() {
result.LastTimestamp = listen.ListenedAt
importResult.UpdateTimestamp(listen.ListenedAt)
importResult.ImportCount += 1
}
result.ImportCount += 1
}
return result, nil
return importResult, nil
}
func (s Scrobble) ToListen() models.Listen {

15
backends/scrobblerlog/parser.go
View file

@ -37,12 +37,12 @@ import (
type ScrobblerLog struct {
Timezone string
Client string
Listens []models.Listen
Listens models.ListensList
}
func Parse(data io.Reader, includeSkipped bool) (ScrobblerLog, error) {
result := ScrobblerLog{
Listens: make([]models.Listen, 0),
Listens: make(models.ListensList, 0),
}
reader := bufio.NewReader(data)
@ -92,16 +92,11 @@ func Parse(data io.Reader, includeSkipped bool) (ScrobblerLog, error) {
return result, nil
}
func Write(data io.Writer, log *ScrobblerLog) (lastTimestamp time.Time, err error) {
err = writeHeader(data, log)
if err != nil {
return
}
func Write(data io.Writer, listens models.ListensList) (lastTimestamp time.Time, err error) {
tsvWriter := csv.NewWriter(data)
tsvWriter.Comma = '\t'
for _, listen := range log.Listens {
for _, listen := range listens {
if listen.ListenedAt.Unix() > lastTimestamp.Unix() {
lastTimestamp = listen.ListenedAt
}
@ -162,7 +157,7 @@ func readHeader(reader *bufio.Reader, log *ScrobblerLog) error {
return nil
}
func writeHeader(writer io.Writer, log *ScrobblerLog) error {
func WriteHeader(writer io.Writer, log *ScrobblerLog) error {
headers := []string{
"#AUDIOSCROBBLER/1.1\n",
"#TZ/" + log.Timezone + "\n",

4
backends/scrobblerlog/parser_test.go
View file

@ -98,7 +98,9 @@ func TestWrite(t *testing.T) {
},
},
}
lastTimestamp, err := scrobblerlog.Write(buffer, &log)
err := scrobblerlog.WriteHeader(buffer, &log)
require.NoError(t, err)
lastTimestamp, err := scrobblerlog.Write(buffer, log.Listens)
require.NoError(t, err)
result := string(buffer.Bytes())
lines := strings.Split(result, "\n")

49
backends/scrobblerlog/scrobblerlog.go
View file

@ -23,6 +23,7 @@ package scrobblerlog
import (
"os"
"sort"
"time"
"github.com/spf13/viper"
@ -40,31 +41,36 @@ func (b ScrobblerLogBackend) FromConfig(config *viper.Viper) models.Backend {
return b
}
func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time) ([]models.Listen, error) {
func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
defer close(results)
file, err := os.Open(b.filePath)
if err != nil {
return nil, err
results <- models.ListensResult{Error: err}
return
}
defer file.Close()
result, err := Parse(file, b.includeSkipped)
log, err := Parse(file, b.includeSkipped)
if err != nil {
return nil, err
results <- models.ListensResult{Error: err}
close(results)
return
}
return result.Listens, nil
listens := log.Listens.NewerThan(oldestTimestamp)
sort.Sort(listens)
results <- models.ListensResult{Listens: listens}
}
func (b ScrobblerLogBackend) ImportListens(listens []models.Listen, oldestTimestamp time.Time) (models.ImportResult, error) {
result := models.ImportResult{
TotalCount: len(listens),
func (b ScrobblerLogBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
}
file, err := os.Create(b.filePath)
if err != nil {
return result, err
return importResult, err
}
defer file.Close()
@ -72,16 +78,27 @@ func (b ScrobblerLogBackend) ImportListens(listens []models.Listen, oldestTimest
log := ScrobblerLog{
Timezone: "UNKNOWN",
Client: "Rockbox unknown $Revision$",
Listens: listens,
}
lastTimestamp, err := Write(file, &log)
err = WriteHeader(file, &log)
if err != nil {
return result, err
return importResult, err
}
result.LastTimestamp = lastTimestamp
result.ImportCount = len(listens)
return result, nil
for result := range results {
if result.Error != nil {
return importResult, result.Error
}
importResult.TotalCount += len(result.Listens)
lastTimestamp, err := Write(file, result.Listens)
if err != nil {
return importResult, err
}
importResult.UpdateTimestamp(lastTimestamp)
importResult.ImportCount += len(result.Listens)
}
return importResult, nil
}

5
backends/subsonic/subsonic.go
View file

@ -48,23 +48,20 @@ func (b SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend {
}
func (b SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
defer close(results)
err := b.client.Authenticate(b.password)
if err != nil {
results <- models.LovesResult{Error: err}
close(results)
return
}
starred, err := b.client.GetStarred2(map[string]string{})
if err != nil {
results <- models.LovesResult{Error: err}
close(results)
return
}
results <- models.LovesResult{Loves: b.filterSongs(starred.Song, oldestTimestamp)}
close(results)
return
}
func (b SubsonicApiBackend) filterSongs(songs []*subsonic.Child, oldestTimestamp time.Time) models.LovesList {

11
models/models.go
View file

@ -67,6 +67,17 @@ type Love struct {
type ListensList []Listen
// Returns a new ListensList with only elements that are newer than t.
func (l ListensList) NewerThan(t time.Time) ListensList {
result := make(ListensList, 0, len(l))
for _, item := range l {
if item.ListenedAt.Unix() > t.Unix() {
result = append(result, item)
}
}
return result
}
func (l ListensList) Len() int {
return len(l)
}

16
models/models_test.go
View file

@ -27,6 +27,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uploadedlobster.com/scotty/models"
)
@ -41,7 +42,7 @@ func TestTrackArtistName(t *testing.T) {
assert.Equal(t, "Foo, Bar, Baz", track.ArtistName())
}
func TestListensListSort(t *testing.T) {
func TestListensListNewerThan(t *testing.T) {
listen1 := models.Listen{ListenedAt: time.Unix(3, 0)}
listen2 := models.Listen{ListenedAt: time.Unix(0, 0)}
listen3 := models.Listen{ListenedAt: time.Unix(2, 0)}
@ -52,6 +53,19 @@ func TestListensListSort(t *testing.T) {
assert.Equal(t, listen3, list[1])
}
func TestListensListSort(t *testing.T) {
now := time.Now()
listen1 := models.Listen{UserName: "l1", ListenedAt: now.Add(-1 * time.Hour)}
listen2 := models.Listen{UserName: "l2", ListenedAt: now}
listen3 := models.Listen{UserName: "l3", ListenedAt: now.Add(1 * time.Hour)}
listen4 := models.Listen{UserName: "l4", ListenedAt: now.Add(2 * time.Hour)}
list := models.ListensList{listen1, listen2, listen3, listen4}
newList := list.NewerThan(now)
require.Len(t, newList, 2)
assert.Equal(t, listen3, newList[0])
assert.Equal(t, listen4, newList[1])
}
func TestLovesListSort(t *testing.T) {
love1 := models.Love{Created: time.Unix(3, 0)}
love2 := models.Love{Created: time.Unix(0, 0)}