marc/scotty
1
0
Fork 0
mirror of https://git.sr.ht/~phw/scotty synced 2025-04-15 17:49:29 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implemented progressbar for export/import

Browse source
This commit is contained in:
Philipp Wolfer 2023-11-16 00:45:00 +01:00
parent ab04eb1123
commit 6e330daf06
No known key found for this signature in database
GPG key ID: 8FDF744D4919943B
24 changed files with 590 additions and 239 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
backends/backends.go
View file

@ -52,7 +52,7 @@ func ResolveBackend[T interface{}](config *viper.Viper) (T, error) {
if err != nil {
return result, err
}
implements, interfaceName := implementsInterface[T](backend)
implements, interfaceName := ImplementsInterface[T](&backend)
if implements {
result = backend.(T)
} else {
@ -91,14 +91,14 @@ func resolveBackend(config *viper.Viper) (string, models.Backend, error) {
backendName := config.GetString("backend")
backendType := knownBackends[backendName]
if backendType == nil {
return backendName, nil, errors.New(fmt.Sprintf("Unknown backend %s", backendName))
return backendName, nil, fmt.Errorf("Unknown backend %s", backendName)
}
return backendName, backendType().FromConfig(config), nil
}
func implementsInterface[T interface{}](backend models.Backend) (bool, string) {
func ImplementsInterface[T interface{}](backend *models.Backend) (bool, string) {
expectedInterface := reflect.TypeOf((*T)(nil)).Elem()
implements := backend != nil && reflect.TypeOf(backend).Implements(expectedInterface)
implements := backend != nil && reflect.TypeOf(*backend).Implements(expectedInterface)
return implements, expectedInterface.Name()
}
@ -133,7 +133,7 @@ func getImportCapabilities(backend models.Backend) []Capability {
}
func checkCapability[T interface{}](backend models.Backend, suffix string) (string, bool) {
implements, name := implementsInterface[T](backend)
implements, name := ImplementsInterface[T](&backend)
if implements {
cap, found := strings.CutSuffix(strings.ToLower(name), suffix)
if found {

45
backends/backends_test.go
View file

@ -23,12 +23,19 @@ THE SOFTWARE.
package backends_test
import (
"reflect"
"testing"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"go.uploadedlobster.com/scotty/backends"
"go.uploadedlobster.com/scotty/backends/dump"
"go.uploadedlobster.com/scotty/backends/funkwhale"
"go.uploadedlobster.com/scotty/backends/jspf"
"go.uploadedlobster.com/scotty/backends/listenbrainz"
"go.uploadedlobster.com/scotty/backends/maloja"
"go.uploadedlobster.com/scotty/backends/scrobblerlog"
"go.uploadedlobster.com/scotty/backends/subsonic"
"go.uploadedlobster.com/scotty/models"
)
@ -37,7 +44,7 @@ func TestResolveBackend(t *testing.T) {
config.Set("backend", "dump")
backend, err := backends.ResolveBackend[models.ListensImport](config)
assert.NoError(t, err)
assert.IsType(t, dump.DumpBackend{}, backend)
assert.IsType(t, &dump.DumpBackend{}, backend)
}
func TestResolveBackendUnknown(t *testing.T) {
@ -69,3 +76,39 @@ func TestGetBackends(t *testing.T) {
// If we got here the "dump" backend was not included
t.Errorf("GetBackends() did not return expected bacend \"dump\"")
}
func TestImplementsInterfaces(t *testing.T) {
expectInterface[models.ListensImport](t, &dump.DumpBackend{})
expectInterface[models.LovesImport](t, &dump.DumpBackend{})
expectInterface[models.ListensExport](t, &funkwhale.FunkwhaleApiBackend{})
// expectInterface[models.ListensImport](t, &funkwhale.FunkwhaleApiBackend{})
expectInterface[models.LovesExport](t, &funkwhale.FunkwhaleApiBackend{})
// expectInterface[models.LovesImport](t, &funkwhale.FunkwhaleApiBackend{})
// expectInterface[models.ListensExport](t, &jspf.JspfBackend{})
// expectInterface[models.ListensImport](t, &jspf.JspfBackend{})
// expectInterface[models.LovesExport](t, &jspf.JspfBackend{})
expectInterface[models.LovesImport](t, &jspf.JspfBackend{})
expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{})
// expectInterface[models.ListensImport](t, &listenbrainz.ListenBrainzApiBackend{})
expectInterface[models.LovesExport](t, &listenbrainz.ListenBrainzApiBackend{})
expectInterface[models.LovesImport](t, &listenbrainz.ListenBrainzApiBackend{})
expectInterface[models.ListensExport](t, &maloja.MalojaApiBackend{})
expectInterface[models.ListensImport](t, &maloja.MalojaApiBackend{})
expectInterface[models.ListensExport](t, &scrobblerlog.ScrobblerLogBackend{})
expectInterface[models.ListensImport](t, &scrobblerlog.ScrobblerLogBackend{})
expectInterface[models.LovesExport](t, &subsonic.SubsonicApiBackend{})
// expectInterface[models.LovesImport](t, &subsonic.SubsonicApiBackend{})
}
func expectInterface[T interface{}](t *testing.T, backend models.Backend) {
ok, name := backends.ImplementsInterface[T](&backend)
if !ok {
t.Errorf("%v expected to implement %v", reflect.TypeOf(backend).Name(), name)
}
}

52
backends/dump/dump.go
View file

@ -22,55 +22,39 @@ THE SOFTWARE.
package dump
import (
"fmt"
"time"
"github.com/spf13/viper"
"go.uploadedlobster.com/scotty/models"
)
type DumpBackend struct{}
func (b DumpBackend) FromConfig(config *viper.Viper) models.Backend {
func (b *DumpBackend) FromConfig(config *viper.Viper) models.Backend {
return b
}
func (b DumpBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
}
for result := range results {
if result.Error != nil {
return importResult, result.Error
}
func (b *DumpBackend) Init() error { return nil }
func (b *DumpBackend) Finish() error { return nil }
importResult.TotalCount += len(result.Listens)
for _, listen := range result.Listens {
importResult.UpdateTimestamp(listen.ListenedAt)
importResult.ImportCount += 1
fmt.Printf("🎶 %v: \"%v\" by %v (%v)\n",
listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMbid)
}
func (b *DumpBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
for _, listen := range export.Listens {
importResult.UpdateTimestamp(listen.ListenedAt)
importResult.ImportCount += 1
progress <- models.Progress{}.FromImportResult(importResult)
// fmt.Printf("🎶 %v: \"%v\" by %v (%v)\n",
// listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMbid)
}
return importResult, nil
}
func (b DumpBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
func (b *DumpBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
for _, love := range export.Loves {
importResult.UpdateTimestamp(love.Created)
importResult.ImportCount += 1
progress <- models.Progress{}.FromImportResult(importResult)
// fmt.Printf("❤️ %v: \"%v\" by %v (%v)\n",
// love.Created, love.TrackName, love.ArtistName(), love.RecordingMbid)
}
for result := range results {
if result.Error != nil {
return importResult, result.Error
}
importResult.TotalCount += len(result.Loves)
for _, love := range result.Loves {
importResult.UpdateTimestamp(love.Created)
importResult.ImportCount += 1
fmt.Printf("❤️ %v: \"%v\" by %v (%v)\n",
love.Created, love.TrackName, love.ArtistName(), love.RecordingMbid)
}
}
return importResult, nil
}

28
backends/funkwhale/funkwhale.go
View file

@ -36,7 +36,7 @@ type FunkwhaleApiBackend struct {
username string
}
func (b FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend {
func (b *FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend {
b.client = NewClient(
config.GetString("server-url"),
config.GetString("token"),
@ -45,19 +45,22 @@ func (b FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend {
return b
}
func (b FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
page := 1
perPage := MaxItemsPerGet
defer close(results)
defer close(progress)
// We need to gather the full list of listens in order to sort them
listens := make(models.ListensList, 0, 2*MaxItemsPerGet)
listens := make(models.ListensList, 0, 2*perPage)
p := models.Progress{Total: int64(perPage)}
out:
for {
result, err := b.client.GetHistoryListenings(b.username, page, perPage)
if err != nil {
results <- models.ListensResult{Error: err}
close(results)
}
count := len(result.Results)
@ -68,6 +71,7 @@ out:
for _, fwListen := range result.Results {
listen := fwListen.ToListen()
if listen.ListenedAt.Unix() > oldestTimestamp.Unix() {
p.Elapsed += 1
listens = append(listens, listen)
} else {
break out
@ -76,25 +80,31 @@ out:
if result.Next == "" {
// No further results
p.Total = p.Elapsed
p.Total -= int64(perPage - count)
break out
}
p.Total += int64(perPage)
progress <- p
page += 1
}
sort.Sort(listens)
progress <- p.Complete()
results <- models.ListensResult{Listens: listens}
close(results)
}
func (b FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
page := 1
perPage := MaxItemsPerGet
defer close(results)
defer close(progress)
// We need to gather the full list of listens in order to sort them
loves := make(models.LovesList, 0, 2*MaxItemsPerGet)
loves := make(models.LovesList, 0, 2*perPage)
p := models.Progress{Total: int64(perPage)}
out:
for {
@ -112,6 +122,7 @@ out:
for _, favorite := range result.Results {
love := favorite.ToLove()
if love.Created.Unix() > oldestTimestamp.Unix() {
p.Elapsed += 1
loves = append(loves, love)
} else {
break out
@ -123,10 +134,13 @@ out:
break out
}
p.Total += int64(perPage)
progress <- p
page += 1
}
sort.Sort(loves)
progress <- p.Complete()
results <- models.LovesResult{Loves: loves}
}

4
backends/funkwhale/funkwhale_test.go
View file

@ -35,8 +35,8 @@ import (
func TestFromConfig(t *testing.T) {
config := viper.New()
config.Set("token", "thetoken")
backend := funkwhale.FunkwhaleApiBackend{}.FromConfig(config)
assert.IsType(t, funkwhale.FunkwhaleApiBackend{}, backend)
backend := (&funkwhale.FunkwhaleApiBackend{}).FromConfig(config)
assert.IsType(t, &funkwhale.FunkwhaleApiBackend{}, backend)
}
func TestFunkwhaleListeningToListen(t *testing.T) {

42
backends/jspf/jspf.go
View file

@ -36,42 +36,34 @@ type JspfBackend struct {
title string
creator string
identifier string
tracks []Track
}
func (b JspfBackend) FromConfig(config *viper.Viper) models.Backend {
func (b *JspfBackend) FromConfig(config *viper.Viper) models.Backend {
b.filePath = config.GetString("file-path")
b.title = config.GetString("title")
b.creator = config.GetString("username")
b.identifier = config.GetString("identifier")
b.tracks = make([]Track, 0)
return b
}
func (b JspfBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
func (b *JspfBackend) Init() error { return nil }
func (b *JspfBackend) Finish() error {
err := b.writeJspf(b.tracks)
return err
}
func (b *JspfBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
for _, love := range export.Loves {
track := loveToTrack(love)
b.tracks = append(b.tracks, track)
importResult.ImportCount += 1
importResult.UpdateTimestamp(love.Created)
}
tracks := make([]Track, 0, importResult.TotalCount)
for result := range results {
if result.Error != nil {
return importResult, result.Error
}
importResult.TotalCount += len(result.Loves)
for _, love := range result.Loves {
track := loveToTrack(love)
tracks = append(tracks, track)
oldestTimestamp = love.Created
importResult.ImportCount += 1
}
}
err := b.writeJspf(tracks)
if err != nil {
importResult.UpdateTimestamp(oldestTimestamp)
importResult.ImportCount = len(tracks)
}
return importResult, err
progress <- models.Progress{}.FromImportResult(importResult)
return importResult, nil
}
func loveToTrack(love models.Love) Track {

4
backends/jspf/jspf_test.go
View file

@ -36,6 +36,6 @@ func TestFromConfig(t *testing.T) {
config.Set("title", "My Playlist")
config.Set("username", "outsidecontext")
config.Set("identifier", "http://example.com/playlist1")
backend := scrobblerlog.ScrobblerLogBackend{}.FromConfig(config)
assert.IsType(t, scrobblerlog.ScrobblerLogBackend{}, backend)
backend := (&scrobblerlog.ScrobblerLogBackend{}).FromConfig(config)
assert.IsType(t, &scrobblerlog.ScrobblerLogBackend{}, backend)
}

144
backends/listenbrainz/listenbrainz.go
View file

@ -31,25 +31,34 @@ import (
)
type ListenBrainzApiBackend struct {
client Client
username string
client Client
username string
existingMbids map[string]bool
}
func (b ListenBrainzApiBackend) FromConfig(config *viper.Viper) models.Backend {
func (b *ListenBrainzApiBackend) FromConfig(config *viper.Viper) models.Backend {
b.client = NewClient(config.GetString("token"))
b.client.MaxResults = MaxItemsPerGet
b.username = config.GetString("username")
return b
}
func (b ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
maxTime := time.Now()
func (b *ListenBrainzApiBackend) Init() error { return nil }
func (b *ListenBrainzApiBackend) Finish() error { return nil }
func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
startTime := time.Now()
maxTime := startTime
minTime := time.Unix(0, 0)
totalDuration := startTime.Sub(oldestTimestamp)
defer close(results)
defer close(progress)
// FIXME: Optimize by fetching the listens in reverse listen time order
listens := make(models.ListensList, 0, 2*MaxItemsPerGet)
p := models.Progress{Total: int64(totalDuration.Seconds())}
out:
for {
@ -66,6 +75,7 @@ out:
// Set maxTime to the oldest returned listen
maxTime = time.Unix(result.Payload.Listens[count-1].ListenedAt, 0)
remainingTime := maxTime.Sub(oldestTimestamp)
for _, listen := range result.Payload.Listens {
if listen.ListenedAt > oldestTimestamp.Unix() {
@ -73,19 +83,26 @@ out:
} else {
// result contains listens older then oldestTimestamp,
// we can stop requesting more
p.Total = int64(startTime.Sub(time.Unix(listen.ListenedAt, 0)).Seconds())
break out
}
}
p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
progress <- p
}
sort.Sort(listens)
results <- models.ListensResult{Listens: listens}
progress <- p.Complete()
results <- models.ListensResult{Listens: listens, OldestTimestamp: oldestTimestamp}
}
func (b ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
offset := 0
defer close(results)
defer close(progress)
loves := make(models.LovesList, 0, 2*MaxItemsPerGet)
p := models.Progress{}
out:
for {
@ -104,84 +121,77 @@ out:
love := feedback.ToLove()
if love.Created.Unix() > oldestTimestamp.Unix() {
loves = append(loves, love)
p.Elapsed += 1
progress <- p
} else {
break out
}
}
p.Total = int64(result.TotalCount)
p.Elapsed += int64(count)
offset += MaxItemsPerGet
}
sort.Sort(loves)
progress <- p.Complete()
results <- models.LovesResult{Loves: loves}
}
func (b ListenBrainzApiBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
ImportErrors: make([]string, 0),
}
existingLovesChan := make(chan models.LovesResult)
go b.ExportLoves(time.Unix(0, 0), existingLovesChan)
existingLoves := <-existingLovesChan
if existingLoves.Error != nil {
results <- models.LovesResult{Error: existingLoves.Error}
close(results)
}
existingMbids := make(map[string]bool, len(existingLoves.Loves))
for _, love := range existingLoves.Loves {
existingMbids[string(love.RecordingMbid)] = true
}
for result := range results {
if result.Error != nil {
return importResult, result.Error
func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
if len(b.existingMbids) == 0 {
existingLovesChan := make(chan models.LovesResult)
go b.ExportLoves(time.Unix(0, 0), existingLovesChan, progress)
existingLoves := <-existingLovesChan
if existingLoves.Error != nil {
return importResult, existingLoves.Error
}
importResult.TotalCount += len(result.Loves)
// TODO: Store MBIDs directly
b.existingMbids = make(map[string]bool, len(existingLoves.Loves))
for _, love := range existingLoves.Loves {
b.existingMbids[string(love.RecordingMbid)] = true
}
}
for _, love := range result.Loves {
if love.Created.Unix() <= oldestTimestamp.Unix() {
continue
}
for _, love := range export.Loves {
recordingMbid := string(love.RecordingMbid)
recordingMbid := string(love.RecordingMbid)
if recordingMbid == "" {
lookup, err := b.client.Lookup(love.TrackName, love.ArtistName())
if err == nil {
recordingMbid = lookup.RecordingMbid
}
}
if recordingMbid != "" {
ok := false
errMsg := ""
if existingMbids[recordingMbid] {
ok = true
} else {
resp, err := b.client.SendFeedback(Feedback{
RecordingMbid: recordingMbid,
Score: 1,
})
ok = err == nil && resp.Status == "ok"
if err != nil {
errMsg = err.Error()
}
}
if ok {
importResult.UpdateTimestamp(love.Created)
importResult.ImportCount += 1
} else {
msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v",
love.TrackName, love.ArtistName(), errMsg)
importResult.ImportErrors = append(importResult.ImportErrors, msg)
}
if recordingMbid == "" {
lookup, err := b.client.Lookup(love.TrackName, love.ArtistName())
if err == nil {
recordingMbid = lookup.RecordingMbid
}
}
if recordingMbid != "" {
ok := false
errMsg := ""
if b.existingMbids[recordingMbid] {
ok = true
} else {
resp, err := b.client.SendFeedback(Feedback{
RecordingMbid: recordingMbid,
Score: 1,
})
ok = err == nil && resp.Status == "ok"
if err != nil {
errMsg = err.Error()
}
}
if ok {
importResult.UpdateTimestamp(love.Created)
importResult.ImportCount += 1
} else {
msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v",
love.TrackName, love.ArtistName(), errMsg)
importResult.ImportErrors = append(importResult.ImportErrors, msg)
}
}
progress <- models.Progress{}.FromImportResult(importResult)
}
return importResult, nil

4
backends/listenbrainz/listenbrainz_test.go
View file

@ -35,8 +35,8 @@ import (
func TestFromConfig(t *testing.T) {
config := viper.New()
config.Set("token", "thetoken")
backend := listenbrainz.ListenBrainzApiBackend{}.FromConfig(config)
assert.IsType(t, listenbrainz.ListenBrainzApiBackend{}, backend)
backend := (&listenbrainz.ListenBrainzApiBackend{}).FromConfig(config)
assert.IsType(t, &listenbrainz.ListenBrainzApiBackend{}, backend)
}
func TestListenBrainzListenToListen(t *testing.T) {

65
backends/maloja/maloja.go
View file

@ -36,7 +36,7 @@ type MalojaApiBackend struct {
nofix bool
}
func (b MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend {
func (b *MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend {
b.client = NewClient(
config.GetString("server-url"),
config.GetString("token"),
@ -45,14 +45,19 @@ func (b MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend {
return b
}
func (b MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
func (b *MalojaApiBackend) Init() error { return nil }
func (b *MalojaApiBackend) Finish() error { return nil }
func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
page := 0
perPage := MaxItemsPerGet
defer close(results)
defer close(progress)
// We need to gather the full list of listens in order to sort them
listens := make(models.ListensList, 0, 2*perPage)
p := models.Progress{Total: int64(perPage)}
out:
for {
@ -69,55 +74,45 @@ out:
for _, scrobble := range result.List {
if scrobble.ListenedAt > oldestTimestamp.Unix() {
p.Elapsed += 1
listens = append(listens, scrobble.ToListen())
} else {
break out
}
}
p.Total += int64(perPage)
progress <- p
page += 1
}
sort.Sort(listens)
progress <- p.Complete()
results <- models.ListensResult{Listens: listens}
}
func (b MalojaApiBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
}
for result := range results {
if result.Error != nil {
return importResult, result.Error
func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
for _, listen := range export.Listens {
scrobble := NewScrobble{
Title: listen.TrackName,
Artists: listen.ArtistNames,
Album: listen.ReleaseName,
Duration: int64(listen.PlaybackDuration.Seconds()),
Length: int64(listen.Duration.Seconds()),
Time: listen.ListenedAt.Unix(),
Nofix: b.nofix,
}
importResult.TotalCount += len(result.Listens)
for _, listen := range result.Listens {
if listen.ListenedAt.Unix() <= oldestTimestamp.Unix() {
break
}
scrobble := NewScrobble{
Title: listen.TrackName,
Artists: listen.ArtistNames,
Album: listen.ReleaseName,
Duration: int64(listen.PlaybackDuration.Seconds()),
Length: int64(listen.Duration.Seconds()),
Time: listen.ListenedAt.Unix(),
Nofix: b.nofix,
}
resp, err := b.client.NewScrobble(scrobble)
if err != nil {
return importResult, err
} else if resp.Status != "success" {
return importResult, errors.New(resp.Error.Description)
}
importResult.UpdateTimestamp(listen.ListenedAt)
importResult.ImportCount += 1
resp, err := b.client.NewScrobble(scrobble)
if err != nil {
return importResult, err
} else if resp.Status != "success" {
return importResult, errors.New(resp.Error.Description)
}
importResult.UpdateTimestamp(listen.ListenedAt)
importResult.ImportCount += 1
progress <- models.Progress{}.FromImportResult(importResult)
}
return importResult, nil

4
backends/maloja/maloja_test.go
View file

@ -33,8 +33,8 @@ import (
func TestFromConfig(t *testing.T) {
config := viper.New()
config.Set("token", "thetoken")
backend := maloja.MalojaApiBackend{}.FromConfig(config)
assert.IsType(t, maloja.MalojaApiBackend{}, backend)
backend := (&maloja.MalojaApiBackend{}).FromConfig(config)
assert.IsType(t, &maloja.MalojaApiBackend{}, backend)
}
func TestScrobbleToListen(t *testing.T) {

107
backends/process.go Normal file
View file

@ -0,0 +1,107 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package backends
import "go.uploadedlobster.com/scotty/models"
func ProcessListensImports(importer models.ListensImport, results chan models.ListensResult, out chan models.ImportResult, progress chan models.Progress) {
defer close(out)
defer close(progress)
result := models.ImportResult{}
err := importer.Init()
if err != nil {
handleError(result, err, out, progress)
return
}
for exportResult := range results {
if exportResult.Error != nil {
handleError(result, exportResult.Error, out, progress)
return
}
result.TotalCount += len(exportResult.Listens)
importResult, err := importer.ImportListens(exportResult, result, progress)
if err != nil {
handleError(importResult, err, out, progress)
return
}
result.Update(importResult)
progress <- models.Progress{}.FromImportResult(result)
}
err = importer.Finish()
if err != nil {
handleError(result, err, out, progress)
return
}
progress <- models.Progress{}.FromImportResult(result).Complete()
out <- result
}
func ProcessLovesImports(importer models.LovesImport, results chan models.LovesResult, out chan models.ImportResult, progress chan models.Progress) {
defer close(out)
defer close(progress)
result := models.ImportResult{}
err := importer.Init()
if err != nil {
handleError(result, err, out, progress)
return
}
for exportResult := range results {
if exportResult.Error != nil {
handleError(result, exportResult.Error, out, progress)
return
}
result.TotalCount += len(exportResult.Loves)
importResult, err := importer.ImportLoves(exportResult, result, progress)
if err != nil {
handleError(importResult, err, out, progress)
return
}
result.Update(importResult)
progress <- models.Progress{}.FromImportResult(result)
}
err = importer.Finish()
if err != nil {
handleError(result, err, out, progress)
return
}
progress <- models.Progress{}.FromImportResult(result).Complete()
out <- result
}
func handleError(result models.ImportResult, err error, out chan models.ImportResult, progress chan models.Progress) {
result.Error = err
progress <- models.Progress{}.FromImportResult(result).Complete()
out <- result
}

70
backends/scrobblerlog/scrobblerlog.go
View file

@ -33,16 +33,44 @@ import (
type ScrobblerLogBackend struct {
filePath string
includeSkipped bool
file *os.File
log ScrobblerLog
}
func (b ScrobblerLogBackend) FromConfig(config *viper.Viper) models.Backend {
func (b *ScrobblerLogBackend) FromConfig(config *viper.Viper) models.Backend {
b.filePath = config.GetString("file-path")
b.includeSkipped = config.GetBool("include-skipped")
return b
}
func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
func (b *ScrobblerLogBackend) Init() error {
file, err := os.Create(b.filePath)
if err != nil {
return err
}
b.log = ScrobblerLog{
Timezone: "UNKNOWN",
Client: "Rockbox unknown $Revision$",
}
err = WriteHeader(file, &b.log)
if err != nil {
file.Close()
return err
}
b.file = file
return nil
}
func (b *ScrobblerLogBackend) Finish() error {
return b.file.Close()
}
func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
defer close(results)
defer close(progress)
file, err := os.Open(b.filePath)
if err != nil {
results <- models.ListensResult{Error: err}
@ -60,45 +88,19 @@ func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results ch
listens := log.Listens.NewerThan(oldestTimestamp)
sort.Sort(listens)
progress <- models.Progress{Elapsed: int64(len(listens))}.Complete()
results <- models.ListensResult{Listens: listens}
}
func (b ScrobblerLogBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
}
file, err := os.Create(b.filePath)
func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
lastTimestamp, err := Write(b.file, export.Listens)
if err != nil {
return importResult, err
}
defer file.Close()
log := ScrobblerLog{
Timezone: "UNKNOWN",
Client: "Rockbox unknown $Revision$",
}
err = WriteHeader(file, &log)
if err != nil {
return importResult, err
}
for result := range results {
if result.Error != nil {
return importResult, result.Error
}
importResult.TotalCount += len(result.Listens)
lastTimestamp, err := Write(file, result.Listens)
if err != nil {
return importResult, err
}
importResult.UpdateTimestamp(lastTimestamp)
importResult.ImportCount += len(result.Listens)
}
importResult.UpdateTimestamp(lastTimestamp)
importResult.ImportCount = len(export.Listens)
progress <- models.Progress{}.FromImportResult(importResult)
return importResult, nil
}

4
backends/scrobblerlog/scrobblerlog_test.go
View file

@ -32,6 +32,6 @@ import (
func TestFromConfig(t *testing.T) {
config := viper.New()
config.Set("token", "thetoken")
backend := scrobblerlog.ScrobblerLogBackend{}.FromConfig(config)
assert.IsType(t, scrobblerlog.ScrobblerLogBackend{}, backend)
backend := (&scrobblerlog.ScrobblerLogBackend{}).FromConfig(config)
assert.IsType(t, &scrobblerlog.ScrobblerLogBackend{}, backend)
}

8
backends/subsonic/subsonic.go
View file

@ -36,7 +36,7 @@ type SubsonicApiBackend struct {
password string
}
func (b SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend {
func (b *SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend {
b.client = subsonic.Client{
Client: &http.Client{},
BaseUrl: config.GetString("server-url"),
@ -47,8 +47,9 @@ func (b SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend {
return b
}
func (b SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
defer close(results)
defer close(progress)
err := b.client.Authenticate(b.password)
if err != nil {
results <- models.LovesResult{Error: err}
@ -61,10 +62,11 @@ func (b SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan
return
}
progress <- models.Progress{Elapsed: int64(len(starred.Song))}.Complete()
results <- models.LovesResult{Loves: b.filterSongs(starred.Song, oldestTimestamp)}
}
func (b SubsonicApiBackend) filterSongs(songs []*subsonic.Child, oldestTimestamp time.Time) models.LovesList {
func (b *SubsonicApiBackend) filterSongs(songs []*subsonic.Child, oldestTimestamp time.Time) models.LovesList {
loves := make(models.LovesList, len(songs))
for i, song := range songs {
love := SongToLove(*song, b.client.User)

4
backends/subsonic/subsonic_test.go
View file

@ -35,8 +35,8 @@ func TestFromConfig(t *testing.T) {
config := viper.New()
config.Set("server-url", "https://subsonic.example.com")
config.Set("token", "thetoken")
backend := subsonic.SubsonicApiBackend{}.FromConfig(config)
assert.IsType(t, subsonic.SubsonicApiBackend{}, backend)
backend := (&subsonic.SubsonicApiBackend{}).FromConfig(config)
assert.IsType(t, &subsonic.SubsonicApiBackend{}, backend)
}
func TestSongToLove(t *testing.T) {

21
cmd/listens.go
View file

@ -23,6 +23,7 @@ package cmd
import (
"fmt"
"sync"
"time"
"github.com/spf13/cobra"
@ -60,15 +61,25 @@ var listensCmd = &cobra.Command{
}
fmt.Printf("From timestamp: %v (%v)\n", timestamp, timestamp.Unix())
// Prepare progress bars
exportProgress := make(chan models.Progress)
importProgress := make(chan models.Progress)
var wg sync.WaitGroup
progress := progressBar(&wg, exportProgress, importProgress)
// Export from source
listens := make(chan models.ListensResult, 1000)
go exportBackend.ExportListens(timestamp, listens)
listensChan := make(chan models.ListensResult, 1000)
go exportBackend.ExportListens(timestamp, listensChan, exportProgress)
// Import into target
result, err := importBackend.ImportListens(listens, timestamp)
if err != nil {
resultChan := make(chan models.ImportResult)
go backends.ProcessListensImports(importBackend, listensChan, resultChan, importProgress)
result := <-resultChan
wg.Wait()
progress.Wait()
if result.Error != nil {
fmt.Printf("Import failed, last reported timestamp was %v (%v)\n", result.LastTimestamp, result.LastTimestamp.Unix())
cobra.CheckErr(err)
cobra.CheckErr(result.Error)
}
fmt.Printf("Imported %v of %v listens into %v.\n",
result.ImportCount, result.TotalCount, targetName)

22
cmd/loves.go
View file

@ -23,6 +23,7 @@ package cmd
import (
"fmt"
"sync"
"time"
"github.com/spf13/cobra"
@ -60,13 +61,26 @@ var lovesCmd = &cobra.Command{
}
fmt.Printf("From timestamp: %v (%v)\n", timestamp, timestamp.Unix())
// Prepare progress bars
exportProgress := make(chan models.Progress)
importProgress := make(chan models.Progress)
var wg sync.WaitGroup
progress := progressBar(&wg, exportProgress, importProgress)
// Export from source
loves := make(chan models.LovesResult, 1000)
go exportBackend.ExportLoves(timestamp, loves)
lovesChan := make(chan models.LovesResult, 1000)
go exportBackend.ExportLoves(timestamp, lovesChan, exportProgress)
// Import into target
result, err := importBackend.ImportLoves(loves, timestamp)
cobra.CheckErr(err)
resultChan := make(chan models.ImportResult)
go backends.ProcessLovesImports(importBackend, lovesChan, resultChan, importProgress)
result := <-resultChan
wg.Wait()
progress.Wait()
if result.Error != nil {
fmt.Printf("Import failed, last reported timestamp was %v (%v)\n", result.LastTimestamp, result.LastTimestamp.Unix())
cobra.CheckErr(result.Error)
}
fmt.Printf("Imported %v of %v loves into %v.\n",
result.ImportCount, result.TotalCount, targetName)

84
cmd/progress.go Normal file
View file

@ -0,0 +1,84 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package cmd
import (
"sync"
"time"
"github.com/fatih/color"
"github.com/vbauerster/mpb/v8"
"github.com/vbauerster/mpb/v8/decor"
"go.uploadedlobster.com/scotty/models"
)
func progressBar(wg *sync.WaitGroup, exportProgress chan models.Progress, importProgress chan models.Progress) *mpb.Progress {
p := mpb.New(
mpb.WithWaitGroup(wg),
mpb.WithOutput(color.Output),
// mpb.WithWidth(64),
mpb.WithAutoRefresh(),
)
exportBar := setupProgressBar(p, "exporting")
importBar := setupProgressBar(p, "importing")
go updateProgressBar(exportBar, wg, exportProgress)
go updateProgressBar(importBar, wg, importProgress)
return p
}
func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar {
green := color.New(color.FgGreen).SprintFunc()
return p.New(0,
mpb.BarStyle(),
mpb.PrependDecorators(
decor.Name(" "),
decor.OnComplete(
decor.Spinner(nil, decor.WC{W: 2, C: decor.DidentRight}),
green("✓ "),
),
decor.Name(name, decor.WCSyncWidthR),
),
mpb.AppendDecorators(
decor.OnComplete(
decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth}),
"done",
),
// decor.OnComplete(decor.Percentage(decor.WC{W: 5, C: decor.DSyncWidthR}), "done"),
decor.Name(" "),
),
)
}
func updateProgressBar(bar *mpb.Bar, wg *sync.WaitGroup, progressChan chan models.Progress) {
wg.Add(1)
defer wg.Done()
lastIterTime := time.Now()
for progress := range progressChan {
oldIterTime := lastIterTime
lastIterTime = time.Now()
bar.EwmaSetCurrent(progress.Elapsed, lastIterTime.Sub(oldIterTime))
bar.SetTotal(progress.Total, progress.Completed)
}
}

8
go.mod
View file

@ -3,9 +3,12 @@ module go.uploadedlobster.com/scotty
go 1.21.1
require (
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/delucks/go-subsonic v0.0.0-20220915164742-2744002c4be5 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/glebarez/go-sqlite v1.21.2 // indirect
github.com/glebarez/sqlite v1.10.0 // indirect
@ -17,11 +20,14 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
@ -33,6 +39,8 @@ require (
github.com/stretchr/objx v0.5.0 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/vbauerster/mpb v3.4.0+incompatible // indirect
github.com/vbauerster/mpb/v8 v8.6.2 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect

19
go.sum
View file

@ -38,6 +38,10 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@ -61,6 +65,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
@ -151,8 +157,13 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
@ -165,6 +176,9 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ=
@ -197,6 +211,10 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/vbauerster/mpb v3.4.0+incompatible h1:mfiiYw87ARaeRW6x5gWwYRUawxaW1tLAD8IceomUCNw=
github.com/vbauerster/mpb v3.4.0+incompatible/go.mod h1:zAHG26FUhVKETRu+MWqYXcI70POlC6N8up9p1dID7SU=
github.com/vbauerster/mpb/v8 v8.6.2 h1:9EhnJGQRtvgDVCychJgR96EDCOqgg2NsMuk5JUcX4DA=
github.com/vbauerster/mpb/v8 v8.6.2/go.mod h1:oVJ7T+dib99kZ/VBjoBaC8aPXiSAihnzuKmotuihyFo=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -353,6 +371,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

29
models/interfaces.go
View file

@ -30,33 +30,54 @@ import (
// A listen service backend.
// All listen services must implement this interface.
type Backend interface {
// Initialize the backend from a config.
FromConfig(config *viper.Viper) Backend
}
type ImportBackend interface {
Backend
// If the backend needs to setup resources before starting to import,
// this can be done here.
Init() error
// The implementation can perform all steps here to finalize the
// export/import and free used resources.
Finish() error
}
// Must be implemented by services supporting the export of listens.
type ListensExport interface {
Backend
// Returns a list of all listens newer then oldestTimestamp.
// The returned list of listens is supposed to be ordered by the
// Listen.ListenedAt timestamp, with the oldest entry first.
ExportListens(oldestTimestamp time.Time, results chan ListensResult)
ExportListens(oldestTimestamp time.Time, results chan ListensResult, progress chan Progress)
}
// Must be implemented by services supporting the import of listens.
type ListensImport interface {
ImportBackend
// Imports the given list of listens.
ImportListens(results chan ListensResult, oldestTimestamp time.Time) (ImportResult, error)
ImportListens(export ListensResult, importResult ImportResult, progress chan Progress) (ImportResult, error)
}
// Must be implemented by services supporting the export of loves.
type LovesExport interface {
Backend
// Returns a list of all loves newer then oldestTimestamp.
// The returned list of listens is supposed to be ordered by the
// Love.Created timestamp, with the oldest entry first.
ExportLoves(oldestTimestamp time.Time, results chan LovesResult)
ExportLoves(oldestTimestamp time.Time, results chan LovesResult, progress chan Progress)
}
// Must be implemented by services supporting the import of loves.
type LovesImport interface {
ImportBackend
// Imports the given list of loves.
ImportLoves(results chan LovesResult, oldestTimestamp time.Time) (ImportResult, error)
ImportLoves(export LovesResult, importResult ImportResult, progress chan Progress) (ImportResult, error)
}

34
models/models.go
View file

@ -105,13 +105,14 @@ func (l LovesList) Swap(i, j int) {
}
type ListensResult struct {
Error error
Listens ListensList
Listens ListensList
OldestTimestamp time.Time
Error error
}
type LovesResult struct {
Error error
Loves LovesList
Error error
}
type ImportResult struct {
@ -119,6 +120,9 @@ type ImportResult struct {
ImportCount int
LastTimestamp time.Time
ImportErrors []string
// Error is only set if an unrecoverable import error occurred
Error error
}
// Sets LastTimestamp to newTime, if newTime is newer than LastTimestamp
@ -127,3 +131,27 @@ func (i *ImportResult) UpdateTimestamp(newTime time.Time) {
i.LastTimestamp = newTime
}
}
func (i *ImportResult) Update(from ImportResult) {
i.TotalCount = from.TotalCount
i.ImportCount += from.ImportCount
i.UpdateTimestamp(from.LastTimestamp)
}
type Progress struct {
Total int64
Elapsed int64
Completed bool
}
func (p Progress) FromImportResult(result ImportResult) Progress {
p.Total = int64(result.TotalCount)
p.Elapsed = int64(result.ImportCount)
return p
}
func (p Progress) Complete() Progress {
p.Total = p.Elapsed
p.Completed = true
return p
}

17
models/models_test.go
View file

@ -77,6 +77,23 @@ func TestLovesListSort(t *testing.T) {
assert.Equal(t, love3, list[1])
}
func TestImportResultUpdate(t *testing.T) {
result := models.ImportResult{
TotalCount: 100,
ImportCount: 20,
LastTimestamp: time.Now(),
}
newResult := models.ImportResult{
TotalCount: 120,
ImportCount: 50,
LastTimestamp: time.Now().Add(1 * time.Hour),
}
result.Update(newResult)
assert.Equal(t, 120, result.TotalCount)
assert.Equal(t, 70, result.ImportCount)
assert.Equal(t, newResult.LastTimestamp, result.LastTimestamp)
}
func TestImportResultUpdateTimestamp(t *testing.T) {
timestamp := time.Now()
i := models.ImportResult{LastTimestamp: timestamp}