Implemented progressbar for export/import

backends/backends.go

@@ -52,7 +52,7 @@ func ResolveBackend[T interface{}](config *viper.Viper) (T, error) {
 	if err != nil {
 		return result, err
 	}
-	implements, interfaceName := implementsInterface[T](backend)
+	implements, interfaceName := ImplementsInterface[T](&backend)
 	if implements {
 		result = backend.(T)
 	} else {
@@ -91,14 +91,14 @@ func resolveBackend(config *viper.Viper) (string, models.Backend, error) {
 	backendName := config.GetString("backend")
 	backendType := knownBackends[backendName]
 	if backendType == nil {
-		return backendName, nil, errors.New(fmt.Sprintf("Unknown backend %s", backendName))
+		return backendName, nil, fmt.Errorf("Unknown backend %s", backendName)
 	}
 	return backendName, backendType().FromConfig(config), nil
 }
 
-func implementsInterface[T interface{}](backend models.Backend) (bool, string) {
+func ImplementsInterface[T interface{}](backend *models.Backend) (bool, string) {
 	expectedInterface := reflect.TypeOf((*T)(nil)).Elem()
-	implements := backend != nil && reflect.TypeOf(backend).Implements(expectedInterface)
+	implements := backend != nil && reflect.TypeOf(*backend).Implements(expectedInterface)
 	return implements, expectedInterface.Name()
 }
 
@@ -133,7 +133,7 @@ func getImportCapabilities(backend models.Backend) []Capability {
 }
 
 func checkCapability[T interface{}](backend models.Backend, suffix string) (string, bool) {
-	implements, name := implementsInterface[T](backend)
+	implements, name := ImplementsInterface[T](&backend)
 	if implements {
 		cap, found := strings.CutSuffix(strings.ToLower(name), suffix)
 		if found {

backends/backends_test.go

@@ -23,12 +23,19 @@ THE SOFTWARE.
 package backends_test
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"go.uploadedlobster.com/scotty/backends"
 	"go.uploadedlobster.com/scotty/backends/dump"
+	"go.uploadedlobster.com/scotty/backends/funkwhale"
+	"go.uploadedlobster.com/scotty/backends/jspf"
+	"go.uploadedlobster.com/scotty/backends/listenbrainz"
+	"go.uploadedlobster.com/scotty/backends/maloja"
+	"go.uploadedlobster.com/scotty/backends/scrobblerlog"
+	"go.uploadedlobster.com/scotty/backends/subsonic"
 	"go.uploadedlobster.com/scotty/models"
 )
 
@@ -37,7 +44,7 @@ func TestResolveBackend(t *testing.T) {
 	config.Set("backend", "dump")
 	backend, err := backends.ResolveBackend[models.ListensImport](config)
 	assert.NoError(t, err)
-	assert.IsType(t, dump.DumpBackend{}, backend)
+	assert.IsType(t, &dump.DumpBackend{}, backend)
 }
 
 func TestResolveBackendUnknown(t *testing.T) {
@@ -69,3 +76,39 @@ func TestGetBackends(t *testing.T) {
 	// If we got here the "dump" backend was not included
 	t.Errorf("GetBackends() did not return expected bacend \"dump\"")
 }
+
+func TestImplementsInterfaces(t *testing.T) {
+	expectInterface[models.ListensImport](t, &dump.DumpBackend{})
+	expectInterface[models.LovesImport](t, &dump.DumpBackend{})
+
+	expectInterface[models.ListensExport](t, &funkwhale.FunkwhaleApiBackend{})
+	// expectInterface[models.ListensImport](t, &funkwhale.FunkwhaleApiBackend{})
+	expectInterface[models.LovesExport](t, &funkwhale.FunkwhaleApiBackend{})
+	// expectInterface[models.LovesImport](t, &funkwhale.FunkwhaleApiBackend{})
+
+	// expectInterface[models.ListensExport](t, &jspf.JspfBackend{})
+	// expectInterface[models.ListensImport](t, &jspf.JspfBackend{})
+	// expectInterface[models.LovesExport](t, &jspf.JspfBackend{})
+	expectInterface[models.LovesImport](t, &jspf.JspfBackend{})
+
+	expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{})
+	// expectInterface[models.ListensImport](t, &listenbrainz.ListenBrainzApiBackend{})
+	expectInterface[models.LovesExport](t, &listenbrainz.ListenBrainzApiBackend{})
+	expectInterface[models.LovesImport](t, &listenbrainz.ListenBrainzApiBackend{})
+
+	expectInterface[models.ListensExport](t, &maloja.MalojaApiBackend{})
+	expectInterface[models.ListensImport](t, &maloja.MalojaApiBackend{})
+
+	expectInterface[models.ListensExport](t, &scrobblerlog.ScrobblerLogBackend{})
+	expectInterface[models.ListensImport](t, &scrobblerlog.ScrobblerLogBackend{})
+
+	expectInterface[models.LovesExport](t, &subsonic.SubsonicApiBackend{})
+	// expectInterface[models.LovesImport](t, &subsonic.SubsonicApiBackend{})
+}
+
+func expectInterface[T interface{}](t *testing.T, backend models.Backend) {
+	ok, name := backends.ImplementsInterface[T](&backend)
+	if !ok {
+		t.Errorf("%v expected to implement %v", reflect.TypeOf(backend).Name(), name)
+	}
+}

backends/dump/dump.go

@@ -22,55 +22,39 @@ THE SOFTWARE.
 package dump
 
 import (
-	"fmt"
-	"time"
-
 	"github.com/spf13/viper"
 	"go.uploadedlobster.com/scotty/models"
 )
 
 type DumpBackend struct{}
 
-func (b DumpBackend) FromConfig(config *viper.Viper) models.Backend {
+func (b *DumpBackend) FromConfig(config *viper.Viper) models.Backend {
 	return b
 }
 
-func (b DumpBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) {
+func (b *DumpBackend) Init() error   { return nil }
-	importResult := models.ImportResult{
+func (b *DumpBackend) Finish() error { return nil }
-		LastTimestamp: oldestTimestamp,
-	}
-	for result := range results {
-		if result.Error != nil {
-			return importResult, result.Error
-		}
 
-		importResult.TotalCount += len(result.Listens)
+func (b *DumpBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
-		for _, listen := range result.Listens {
+	for _, listen := range export.Listens {
-			importResult.UpdateTimestamp(listen.ListenedAt)
+		importResult.UpdateTimestamp(listen.ListenedAt)
-			importResult.ImportCount += 1
+		importResult.ImportCount += 1
-			fmt.Printf("🎶 %v: \"%v\" by %v (%v)\n",
+		progress <- models.Progress{}.FromImportResult(importResult)
-				listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMbid)
+		// fmt.Printf("🎶 %v: \"%v\" by %v (%v)\n",
-		}
+		// 	listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMbid)
 	}
+
 	return importResult, nil
 }
 
-func (b DumpBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
+func (b *DumpBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
-	importResult := models.ImportResult{
+	for _, love := range export.Loves {
-		LastTimestamp: oldestTimestamp,
+		importResult.UpdateTimestamp(love.Created)
+		importResult.ImportCount += 1
+		progress <- models.Progress{}.FromImportResult(importResult)
+		// fmt.Printf("❤️ %v: \"%v\" by %v (%v)\n",
+		// 	love.Created, love.TrackName, love.ArtistName(), love.RecordingMbid)
 	}
-	for result := range results {
-		if result.Error != nil {
-			return importResult, result.Error
-		}
 
-		importResult.TotalCount += len(result.Loves)
-		for _, love := range result.Loves {
-			importResult.UpdateTimestamp(love.Created)
-			importResult.ImportCount += 1
-			fmt.Printf("❤️ %v: \"%v\" by %v (%v)\n",
-				love.Created, love.TrackName, love.ArtistName(), love.RecordingMbid)
-		}
-	}
 	return importResult, nil
 }

backends/funkwhale/funkwhale.go

@@ -36,7 +36,7 @@ type FunkwhaleApiBackend struct {
 	username string
 }
 
-func (b FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend {
+func (b *FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend {
 	b.client = NewClient(
 		config.GetString("server-url"),
 		config.GetString("token"),
@@ -45,19 +45,22 @@ func (b FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend {
 	return b
 }
 
-func (b FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
+func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
 	page := 1
 	perPage := MaxItemsPerGet
 
+	defer close(results)
+	defer close(progress)
+
 	// We need to gather the full list of listens in order to sort them
-	listens := make(models.ListensList, 0, 2*MaxItemsPerGet)
+	listens := make(models.ListensList, 0, 2*perPage)
+	p := models.Progress{Total: int64(perPage)}
 
 out:
 	for {
 		result, err := b.client.GetHistoryListenings(b.username, page, perPage)
 		if err != nil {
 			results <- models.ListensResult{Error: err}
-			close(results)
 		}
 
 		count := len(result.Results)
@@ -68,6 +71,7 @@ out:
 		for _, fwListen := range result.Results {
 			listen := fwListen.ToListen()
 			if listen.ListenedAt.Unix() > oldestTimestamp.Unix() {
+				p.Elapsed += 1
 				listens = append(listens, listen)
 			} else {
 				break out
@@ -76,25 +80,31 @@ out:
 
 		if result.Next == "" {
 			// No further results
+			p.Total = p.Elapsed
+			p.Total -= int64(perPage - count)
 			break out
 		}
 
+		p.Total += int64(perPage)
+		progress <- p
 		page += 1
 	}
 
 	sort.Sort(listens)
+	progress <- p.Complete()
 	results <- models.ListensResult{Listens: listens}
-	close(results)
 }
 
-func (b FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
+func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
 	page := 1
 	perPage := MaxItemsPerGet
 
 	defer close(results)
+	defer close(progress)
 
 	// We need to gather the full list of listens in order to sort them
-	loves := make(models.LovesList, 0, 2*MaxItemsPerGet)
+	loves := make(models.LovesList, 0, 2*perPage)
+	p := models.Progress{Total: int64(perPage)}
 
 out:
 	for {
@@ -112,6 +122,7 @@ out:
 		for _, favorite := range result.Results {
 			love := favorite.ToLove()
 			if love.Created.Unix() > oldestTimestamp.Unix() {
+				p.Elapsed += 1
 				loves = append(loves, love)
 			} else {
 				break out
@@ -123,10 +134,13 @@ out:
 			break out
 		}
 
+		p.Total += int64(perPage)
+		progress <- p
 		page += 1
 	}
 
 	sort.Sort(loves)
+	progress <- p.Complete()
 	results <- models.LovesResult{Loves: loves}
 }
 

backends/funkwhale/funkwhale_test.go

@@ -35,8 +35,8 @@ import (
 func TestFromConfig(t *testing.T) {
 	config := viper.New()
 	config.Set("token", "thetoken")
-	backend := funkwhale.FunkwhaleApiBackend{}.FromConfig(config)
+	backend := (&funkwhale.FunkwhaleApiBackend{}).FromConfig(config)
-	assert.IsType(t, funkwhale.FunkwhaleApiBackend{}, backend)
+	assert.IsType(t, &funkwhale.FunkwhaleApiBackend{}, backend)
 }
 
 func TestFunkwhaleListeningToListen(t *testing.T) {

backends/jspf/jspf.go

@@ -36,42 +36,34 @@ type JspfBackend struct {
 	title      string
 	creator    string
 	identifier string
+	tracks     []Track
 }
 
-func (b JspfBackend) FromConfig(config *viper.Viper) models.Backend {
+func (b *JspfBackend) FromConfig(config *viper.Viper) models.Backend {
 	b.filePath = config.GetString("file-path")
 	b.title = config.GetString("title")
 	b.creator = config.GetString("username")
 	b.identifier = config.GetString("identifier")
+	b.tracks = make([]Track, 0)
 	return b
 }
 
-func (b JspfBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
+func (b *JspfBackend) Init() error { return nil }
-	importResult := models.ImportResult{
+func (b *JspfBackend) Finish() error {
-		LastTimestamp: oldestTimestamp,
+	err := b.writeJspf(b.tracks)
+	return err
+}
+
+func (b *JspfBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
+	for _, love := range export.Loves {
+		track := loveToTrack(love)
+		b.tracks = append(b.tracks, track)
+		importResult.ImportCount += 1
+		importResult.UpdateTimestamp(love.Created)
 	}
 
-	tracks := make([]Track, 0, importResult.TotalCount)
+	progress <- models.Progress{}.FromImportResult(importResult)
-	for result := range results {
+	return importResult, nil
-		if result.Error != nil {
-			return importResult, result.Error
-		}
-
-		importResult.TotalCount += len(result.Loves)
-		for _, love := range result.Loves {
-			track := loveToTrack(love)
-			tracks = append(tracks, track)
-			oldestTimestamp = love.Created
-			importResult.ImportCount += 1
-		}
-	}
-
-	err := b.writeJspf(tracks)
-	if err != nil {
-		importResult.UpdateTimestamp(oldestTimestamp)
-		importResult.ImportCount = len(tracks)
-	}
-	return importResult, err
 }
 
 func loveToTrack(love models.Love) Track {

backends/jspf/jspf_test.go

@@ -36,6 +36,6 @@ func TestFromConfig(t *testing.T) {
 	config.Set("title", "My Playlist")
 	config.Set("username", "outsidecontext")
 	config.Set("identifier", "http://example.com/playlist1")
-	backend := scrobblerlog.ScrobblerLogBackend{}.FromConfig(config)
+	backend := (&scrobblerlog.ScrobblerLogBackend{}).FromConfig(config)
-	assert.IsType(t, scrobblerlog.ScrobblerLogBackend{}, backend)
+	assert.IsType(t, &scrobblerlog.ScrobblerLogBackend{}, backend)
 }

backends/listenbrainz/listenbrainz.go

@@ -31,25 +31,34 @@ import (
 )
 
 type ListenBrainzApiBackend struct {
-	client   Client
+	client        Client
-	username string
+	username      string
+	existingMbids map[string]bool
 }
 
-func (b ListenBrainzApiBackend) FromConfig(config *viper.Viper) models.Backend {
+func (b *ListenBrainzApiBackend) FromConfig(config *viper.Viper) models.Backend {
 	b.client = NewClient(config.GetString("token"))
 	b.client.MaxResults = MaxItemsPerGet
 	b.username = config.GetString("username")
 	return b
 }
 
-func (b ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
+func (b *ListenBrainzApiBackend) Init() error   { return nil }
-	maxTime := time.Now()
+func (b *ListenBrainzApiBackend) Finish() error { return nil }
+
+func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
+	startTime := time.Now()
+	maxTime := startTime
 	minTime := time.Unix(0, 0)
 
+	totalDuration := startTime.Sub(oldestTimestamp)
+
 	defer close(results)
+	defer close(progress)
 
 	// FIXME: Optimize by fetching the listens in reverse listen time order
 	listens := make(models.ListensList, 0, 2*MaxItemsPerGet)
+	p := models.Progress{Total: int64(totalDuration.Seconds())}
 
 out:
 	for {
@@ -66,6 +75,7 @@ out:
 
 		// Set maxTime to the oldest returned listen
 		maxTime = time.Unix(result.Payload.Listens[count-1].ListenedAt, 0)
+		remainingTime := maxTime.Sub(oldestTimestamp)
 
 		for _, listen := range result.Payload.Listens {
 			if listen.ListenedAt > oldestTimestamp.Unix() {
@@ -73,19 +83,26 @@ out:
 			} else {
 				// result contains listens older then oldestTimestamp,
 				// we can stop requesting more
+				p.Total = int64(startTime.Sub(time.Unix(listen.ListenedAt, 0)).Seconds())
 				break out
 			}
 		}
+
+		p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
+		progress <- p
 	}
 
 	sort.Sort(listens)
-	results <- models.ListensResult{Listens: listens}
+	progress <- p.Complete()
+	results <- models.ListensResult{Listens: listens, OldestTimestamp: oldestTimestamp}
 }
 
-func (b ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
+func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
 	offset := 0
 	defer close(results)
+	defer close(progress)
 	loves := make(models.LovesList, 0, 2*MaxItemsPerGet)
+	p := models.Progress{}
 
 out:
 	for {
@@ -104,84 +121,77 @@ out:
 			love := feedback.ToLove()
 			if love.Created.Unix() > oldestTimestamp.Unix() {
 				loves = append(loves, love)
+				p.Elapsed += 1
+				progress <- p
 			} else {
 				break out
 			}
 		}
 
+		p.Total = int64(result.TotalCount)
+		p.Elapsed += int64(count)
+
 		offset += MaxItemsPerGet
 	}
 
 	sort.Sort(loves)
+	progress <- p.Complete()
 	results <- models.LovesResult{Loves: loves}
 }
 
-func (b ListenBrainzApiBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
+func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
-	importResult := models.ImportResult{
+	if len(b.existingMbids) == 0 {
-		LastTimestamp: oldestTimestamp,
+		existingLovesChan := make(chan models.LovesResult)
-		ImportErrors:  make([]string, 0),
+		go b.ExportLoves(time.Unix(0, 0), existingLovesChan, progress)
-	}
+		existingLoves := <-existingLovesChan
-
+		if existingLoves.Error != nil {
-	existingLovesChan := make(chan models.LovesResult)
+			return importResult, existingLoves.Error
-	go b.ExportLoves(time.Unix(0, 0), existingLovesChan)
-	existingLoves := <-existingLovesChan
-	if existingLoves.Error != nil {
-		results <- models.LovesResult{Error: existingLoves.Error}
-		close(results)
-	}
-
-	existingMbids := make(map[string]bool, len(existingLoves.Loves))
-	for _, love := range existingLoves.Loves {
-		existingMbids[string(love.RecordingMbid)] = true
-	}
-
-	for result := range results {
-		if result.Error != nil {
-			return importResult, result.Error
 		}
 
-		importResult.TotalCount += len(result.Loves)
+		// TODO: Store MBIDs directly
+		b.existingMbids = make(map[string]bool, len(existingLoves.Loves))
+		for _, love := range existingLoves.Loves {
+			b.existingMbids[string(love.RecordingMbid)] = true
+		}
+	}
 
-		for _, love := range result.Loves {
+	for _, love := range export.Loves {
-			if love.Created.Unix() <= oldestTimestamp.Unix() {
+		recordingMbid := string(love.RecordingMbid)
-				continue
-			}
 
-			recordingMbid := string(love.RecordingMbid)
+		if recordingMbid == "" {
-
+			lookup, err := b.client.Lookup(love.TrackName, love.ArtistName())
-			if recordingMbid == "" {
+			if err == nil {
-				lookup, err := b.client.Lookup(love.TrackName, love.ArtistName())
+				recordingMbid = lookup.RecordingMbid
-				if err == nil {
-					recordingMbid = lookup.RecordingMbid
-				}
-			}
-
-			if recordingMbid != "" {
-				ok := false
-				errMsg := ""
-				if existingMbids[recordingMbid] {
-					ok = true
-				} else {
-					resp, err := b.client.SendFeedback(Feedback{
-						RecordingMbid: recordingMbid,
-						Score:         1,
-					})
-					ok = err == nil && resp.Status == "ok"
-					if err != nil {
-						errMsg = err.Error()
-					}
-				}
-
-				if ok {
-					importResult.UpdateTimestamp(love.Created)
-					importResult.ImportCount += 1
-				} else {
-					msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v",
-						love.TrackName, love.ArtistName(), errMsg)
-					importResult.ImportErrors = append(importResult.ImportErrors, msg)
-				}
 			}
 		}
+
+		if recordingMbid != "" {
+			ok := false
+			errMsg := ""
+			if b.existingMbids[recordingMbid] {
+				ok = true
+			} else {
+				resp, err := b.client.SendFeedback(Feedback{
+					RecordingMbid: recordingMbid,
+					Score:         1,
+				})
+				ok = err == nil && resp.Status == "ok"
+				if err != nil {
+					errMsg = err.Error()
+				}
+			}
+
+			if ok {
+				importResult.UpdateTimestamp(love.Created)
+				importResult.ImportCount += 1
+			} else {
+				msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v",
+					love.TrackName, love.ArtistName(), errMsg)
+				importResult.ImportErrors = append(importResult.ImportErrors, msg)
+			}
+		}
+
+		progress <- models.Progress{}.FromImportResult(importResult)
 	}
 
 	return importResult, nil

backends/listenbrainz/listenbrainz_test.go

@@ -35,8 +35,8 @@ import (
 func TestFromConfig(t *testing.T) {
 	config := viper.New()
 	config.Set("token", "thetoken")
-	backend := listenbrainz.ListenBrainzApiBackend{}.FromConfig(config)
+	backend := (&listenbrainz.ListenBrainzApiBackend{}).FromConfig(config)
-	assert.IsType(t, listenbrainz.ListenBrainzApiBackend{}, backend)
+	assert.IsType(t, &listenbrainz.ListenBrainzApiBackend{}, backend)
 }
 
 func TestListenBrainzListenToListen(t *testing.T) {

backends/maloja/maloja.go

@@ -36,7 +36,7 @@ type MalojaApiBackend struct {
 	nofix  bool
 }
 
-func (b MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend {
+func (b *MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend {
 	b.client = NewClient(
 		config.GetString("server-url"),
 		config.GetString("token"),
@@ -45,14 +45,19 @@ func (b MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend {
 	return b
 }
 
-func (b MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
+func (b *MalojaApiBackend) Init() error   { return nil }
+func (b *MalojaApiBackend) Finish() error { return nil }
+
+func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
 	page := 0
 	perPage := MaxItemsPerGet
 
 	defer close(results)
+	defer close(progress)
 
 	// We need to gather the full list of listens in order to sort them
 	listens := make(models.ListensList, 0, 2*perPage)
+	p := models.Progress{Total: int64(perPage)}
 
 out:
 	for {
@@ -69,55 +74,45 @@ out:
 
 		for _, scrobble := range result.List {
 			if scrobble.ListenedAt > oldestTimestamp.Unix() {
+				p.Elapsed += 1
 				listens = append(listens, scrobble.ToListen())
 			} else {
 				break out
 			}
 		}
 
+		p.Total += int64(perPage)
+		progress <- p
 		page += 1
 	}
 
 	sort.Sort(listens)
+	progress <- p.Complete()
 	results <- models.ListensResult{Listens: listens}
 }
 
-func (b MalojaApiBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) {
+func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
-	importResult := models.ImportResult{
+	for _, listen := range export.Listens {
-		LastTimestamp: oldestTimestamp,
+		scrobble := NewScrobble{
-	}
+			Title:    listen.TrackName,
-
+			Artists:  listen.ArtistNames,
-	for result := range results {
+			Album:    listen.ReleaseName,
-		if result.Error != nil {
+			Duration: int64(listen.PlaybackDuration.Seconds()),
-			return importResult, result.Error
+			Length:   int64(listen.Duration.Seconds()),
+			Time:     listen.ListenedAt.Unix(),
+			Nofix:    b.nofix,
 		}
 
-		importResult.TotalCount += len(result.Listens)
+		resp, err := b.client.NewScrobble(scrobble)
-		for _, listen := range result.Listens {
+		if err != nil {
-			if listen.ListenedAt.Unix() <= oldestTimestamp.Unix() {
+			return importResult, err
-				break
+		} else if resp.Status != "success" {
-			}
+			return importResult, errors.New(resp.Error.Description)
-
-			scrobble := NewScrobble{
-				Title:    listen.TrackName,
-				Artists:  listen.ArtistNames,
-				Album:    listen.ReleaseName,
-				Duration: int64(listen.PlaybackDuration.Seconds()),
-				Length:   int64(listen.Duration.Seconds()),
-				Time:     listen.ListenedAt.Unix(),
-				Nofix:    b.nofix,
-			}
-
-			resp, err := b.client.NewScrobble(scrobble)
-			if err != nil {
-				return importResult, err
-			} else if resp.Status != "success" {
-				return importResult, errors.New(resp.Error.Description)
-			}
-
-			importResult.UpdateTimestamp(listen.ListenedAt)
-			importResult.ImportCount += 1
 		}
+
+		importResult.UpdateTimestamp(listen.ListenedAt)
+		importResult.ImportCount += 1
+		progress <- models.Progress{}.FromImportResult(importResult)
 	}
 
 	return importResult, nil

backends/maloja/maloja_test.go

@@ -33,8 +33,8 @@ import (
 func TestFromConfig(t *testing.T) {
 	config := viper.New()
 	config.Set("token", "thetoken")
-	backend := maloja.MalojaApiBackend{}.FromConfig(config)
+	backend := (&maloja.MalojaApiBackend{}).FromConfig(config)
-	assert.IsType(t, maloja.MalojaApiBackend{}, backend)
+	assert.IsType(t, &maloja.MalojaApiBackend{}, backend)
 }
 
 func TestScrobbleToListen(t *testing.T) {

backends/process.go

@@ -0,0 +1,107 @@
+/*
+Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+package backends
+
+import "go.uploadedlobster.com/scotty/models"
+
+func ProcessListensImports(importer models.ListensImport, results chan models.ListensResult, out chan models.ImportResult, progress chan models.Progress) {
+	defer close(out)
+	defer close(progress)
+	result := models.ImportResult{}
+
+	err := importer.Init()
+	if err != nil {
+		handleError(result, err, out, progress)
+		return
+	}
+
+	for exportResult := range results {
+		if exportResult.Error != nil {
+			handleError(result, exportResult.Error, out, progress)
+			return
+		}
+
+		result.TotalCount += len(exportResult.Listens)
+		importResult, err := importer.ImportListens(exportResult, result, progress)
+		if err != nil {
+			handleError(importResult, err, out, progress)
+			return
+		}
+
+		result.Update(importResult)
+		progress <- models.Progress{}.FromImportResult(result)
+	}
+
+	err = importer.Finish()
+	if err != nil {
+		handleError(result, err, out, progress)
+		return
+	}
+
+	progress <- models.Progress{}.FromImportResult(result).Complete()
+	out <- result
+}
+
+func ProcessLovesImports(importer models.LovesImport, results chan models.LovesResult, out chan models.ImportResult, progress chan models.Progress) {
+	defer close(out)
+	defer close(progress)
+	result := models.ImportResult{}
+
+	err := importer.Init()
+	if err != nil {
+		handleError(result, err, out, progress)
+		return
+	}
+
+	for exportResult := range results {
+		if exportResult.Error != nil {
+			handleError(result, exportResult.Error, out, progress)
+			return
+		}
+
+		result.TotalCount += len(exportResult.Loves)
+		importResult, err := importer.ImportLoves(exportResult, result, progress)
+		if err != nil {
+			handleError(importResult, err, out, progress)
+			return
+		}
+
+		result.Update(importResult)
+		progress <- models.Progress{}.FromImportResult(result)
+	}
+
+	err = importer.Finish()
+	if err != nil {
+		handleError(result, err, out, progress)
+		return
+	}
+
+	progress <- models.Progress{}.FromImportResult(result).Complete()
+	out <- result
+}
+
+func handleError(result models.ImportResult, err error, out chan models.ImportResult, progress chan models.Progress) {
+	result.Error = err
+	progress <- models.Progress{}.FromImportResult(result).Complete()
+	out <- result
+}

backends/scrobblerlog/scrobblerlog.go

@@ -33,16 +33,44 @@ import (
 type ScrobblerLogBackend struct {
 	filePath       string
 	includeSkipped bool
+	file           *os.File
+	log            ScrobblerLog
 }
 
-func (b ScrobblerLogBackend) FromConfig(config *viper.Viper) models.Backend {
+func (b *ScrobblerLogBackend) FromConfig(config *viper.Viper) models.Backend {
 	b.filePath = config.GetString("file-path")
 	b.includeSkipped = config.GetBool("include-skipped")
 	return b
 }
 
-func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
+func (b *ScrobblerLogBackend) Init() error {
+	file, err := os.Create(b.filePath)
+	if err != nil {
+		return err
+	}
+
+	b.log = ScrobblerLog{
+		Timezone: "UNKNOWN",
+		Client:   "Rockbox unknown $Revision$",
+	}
+
+	err = WriteHeader(file, &b.log)
+	if err != nil {
+		file.Close()
+		return err
+	}
+
+	b.file = file
+	return nil
+}
+
+func (b *ScrobblerLogBackend) Finish() error {
+	return b.file.Close()
+}
+
+func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
 	defer close(results)
+	defer close(progress)
 	file, err := os.Open(b.filePath)
 	if err != nil {
 		results <- models.ListensResult{Error: err}
@@ -60,45 +88,19 @@ func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results ch
 
 	listens := log.Listens.NewerThan(oldestTimestamp)
 	sort.Sort(listens)
+	progress <- models.Progress{Elapsed: int64(len(listens))}.Complete()
 	results <- models.ListensResult{Listens: listens}
 }
 
-func (b ScrobblerLogBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) {
+func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
-	importResult := models.ImportResult{
+	lastTimestamp, err := Write(b.file, export.Listens)
-		LastTimestamp: oldestTimestamp,
-	}
-
-	file, err := os.Create(b.filePath)
 	if err != nil {
 		return importResult, err
 	}
 
-	defer file.Close()
+	importResult.UpdateTimestamp(lastTimestamp)
-
+	importResult.ImportCount = len(export.Listens)
-	log := ScrobblerLog{
+	progress <- models.Progress{}.FromImportResult(importResult)
-		Timezone: "UNKNOWN",
-		Client:   "Rockbox unknown $Revision$",
-	}
-
-	err = WriteHeader(file, &log)
-	if err != nil {
-		return importResult, err
-	}
-
-	for result := range results {
-		if result.Error != nil {
-			return importResult, result.Error
-		}
-
-		importResult.TotalCount += len(result.Listens)
-		lastTimestamp, err := Write(file, result.Listens)
-		if err != nil {
-			return importResult, err
-		}
-
-		importResult.UpdateTimestamp(lastTimestamp)
-		importResult.ImportCount += len(result.Listens)
-	}
 
 	return importResult, nil
 }

backends/scrobblerlog/scrobblerlog_test.go

@@ -32,6 +32,6 @@ import (
 func TestFromConfig(t *testing.T) {
 	config := viper.New()
 	config.Set("token", "thetoken")
-	backend := scrobblerlog.ScrobblerLogBackend{}.FromConfig(config)
+	backend := (&scrobblerlog.ScrobblerLogBackend{}).FromConfig(config)
-	assert.IsType(t, scrobblerlog.ScrobblerLogBackend{}, backend)
+	assert.IsType(t, &scrobblerlog.ScrobblerLogBackend{}, backend)
 }

backends/subsonic/subsonic.go

@@ -36,7 +36,7 @@ type SubsonicApiBackend struct {
 	password string
 }
 
-func (b SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend {
+func (b *SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend {
 	b.client = subsonic.Client{
 		Client:     &http.Client{},
 		BaseUrl:    config.GetString("server-url"),
@@ -47,8 +47,9 @@ func (b SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend {
 	return b
 }
 
-func (b SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
+func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
 	defer close(results)
+	defer close(progress)
 	err := b.client.Authenticate(b.password)
 	if err != nil {
 		results <- models.LovesResult{Error: err}
@@ -61,10 +62,11 @@ func (b SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan
 		return
 	}
 
+	progress <- models.Progress{Elapsed: int64(len(starred.Song))}.Complete()
 	results <- models.LovesResult{Loves: b.filterSongs(starred.Song, oldestTimestamp)}
 }
 
-func (b SubsonicApiBackend) filterSongs(songs []*subsonic.Child, oldestTimestamp time.Time) models.LovesList {
+func (b *SubsonicApiBackend) filterSongs(songs []*subsonic.Child, oldestTimestamp time.Time) models.LovesList {
 	loves := make(models.LovesList, len(songs))
 	for i, song := range songs {
 		love := SongToLove(*song, b.client.User)

backends/subsonic/subsonic_test.go

@@ -35,8 +35,8 @@ func TestFromConfig(t *testing.T) {
 	config := viper.New()
 	config.Set("server-url", "https://subsonic.example.com")
 	config.Set("token", "thetoken")
-	backend := subsonic.SubsonicApiBackend{}.FromConfig(config)
+	backend := (&subsonic.SubsonicApiBackend{}).FromConfig(config)
-	assert.IsType(t, subsonic.SubsonicApiBackend{}, backend)
+	assert.IsType(t, &subsonic.SubsonicApiBackend{}, backend)
 }
 
 func TestSongToLove(t *testing.T) {

cmd/listens.go

@@ -23,6 +23,7 @@ package cmd
 
 import (
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -60,15 +61,25 @@ var listensCmd = &cobra.Command{
 		}
 		fmt.Printf("From timestamp: %v (%v)\n", timestamp, timestamp.Unix())
 
+		// Prepare progress bars
+		exportProgress := make(chan models.Progress)
+		importProgress := make(chan models.Progress)
+		var wg sync.WaitGroup
+		progress := progressBar(&wg, exportProgress, importProgress)
+
 		// Export from source
-		listens := make(chan models.ListensResult, 1000)
+		listensChan := make(chan models.ListensResult, 1000)
-		go exportBackend.ExportListens(timestamp, listens)
+		go exportBackend.ExportListens(timestamp, listensChan, exportProgress)
 
 		// Import into target
-		result, err := importBackend.ImportListens(listens, timestamp)
+		resultChan := make(chan models.ImportResult)
-		if err != nil {
+		go backends.ProcessListensImports(importBackend, listensChan, resultChan, importProgress)
+		result := <-resultChan
+		wg.Wait()
+		progress.Wait()
+		if result.Error != nil {
 			fmt.Printf("Import failed, last reported timestamp was %v (%v)\n", result.LastTimestamp, result.LastTimestamp.Unix())
-			cobra.CheckErr(err)
+			cobra.CheckErr(result.Error)
 		}
 		fmt.Printf("Imported %v of %v listens into %v.\n",
 			result.ImportCount, result.TotalCount, targetName)

cmd/loves.go

@@ -23,6 +23,7 @@ package cmd
 
 import (
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -60,13 +61,26 @@ var lovesCmd = &cobra.Command{
 		}
 		fmt.Printf("From timestamp: %v (%v)\n", timestamp, timestamp.Unix())
 
+		// Prepare progress bars
+		exportProgress := make(chan models.Progress)
+		importProgress := make(chan models.Progress)
+		var wg sync.WaitGroup
+		progress := progressBar(&wg, exportProgress, importProgress)
+
 		// Export from source
-		loves := make(chan models.LovesResult, 1000)
+		lovesChan := make(chan models.LovesResult, 1000)
-		go exportBackend.ExportLoves(timestamp, loves)
+		go exportBackend.ExportLoves(timestamp, lovesChan, exportProgress)
 
 		// Import into target
-		result, err := importBackend.ImportLoves(loves, timestamp)
+		resultChan := make(chan models.ImportResult)
-		cobra.CheckErr(err)
+		go backends.ProcessLovesImports(importBackend, lovesChan, resultChan, importProgress)
+		result := <-resultChan
+		wg.Wait()
+		progress.Wait()
+		if result.Error != nil {
+			fmt.Printf("Import failed, last reported timestamp was %v (%v)\n", result.LastTimestamp, result.LastTimestamp.Unix())
+			cobra.CheckErr(result.Error)
+		}
 		fmt.Printf("Imported %v of %v loves into %v.\n",
 			result.ImportCount, result.TotalCount, targetName)
 

cmd/progress.go

@@ -0,0 +1,84 @@
+/*
+Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+package cmd
+
+import (
+	"sync"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/vbauerster/mpb/v8"
+	"github.com/vbauerster/mpb/v8/decor"
+	"go.uploadedlobster.com/scotty/models"
+)
+
+func progressBar(wg *sync.WaitGroup, exportProgress chan models.Progress, importProgress chan models.Progress) *mpb.Progress {
+	p := mpb.New(
+		mpb.WithWaitGroup(wg),
+		mpb.WithOutput(color.Output),
+		// mpb.WithWidth(64),
+		mpb.WithAutoRefresh(),
+	)
+
+	exportBar := setupProgressBar(p, "exporting")
+	importBar := setupProgressBar(p, "importing")
+	go updateProgressBar(exportBar, wg, exportProgress)
+	go updateProgressBar(importBar, wg, importProgress)
+
+	return p
+}
+
+func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar {
+	green := color.New(color.FgGreen).SprintFunc()
+	return p.New(0,
+		mpb.BarStyle(),
+		mpb.PrependDecorators(
+			decor.Name(" "),
+			decor.OnComplete(
+				decor.Spinner(nil, decor.WC{W: 2, C: decor.DidentRight}),
+				green("✓ "),
+			),
+			decor.Name(name, decor.WCSyncWidthR),
+		),
+		mpb.AppendDecorators(
+			decor.OnComplete(
+				decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth}),
+				"done",
+			),
+			// decor.OnComplete(decor.Percentage(decor.WC{W: 5, C: decor.DSyncWidthR}), "done"),
+			decor.Name(" "),
+		),
+	)
+}
+
+func updateProgressBar(bar *mpb.Bar, wg *sync.WaitGroup, progressChan chan models.Progress) {
+	wg.Add(1)
+	defer wg.Done()
+	lastIterTime := time.Now()
+	for progress := range progressChan {
+		oldIterTime := lastIterTime
+		lastIterTime = time.Now()
+		bar.EwmaSetCurrent(progress.Elapsed, lastIterTime.Sub(oldIterTime))
+		bar.SetTotal(progress.Total, progress.Completed)
+	}
+}

go.mod

@@ -3,9 +3,12 @@ module go.uploadedlobster.com/scotty
 go 1.21.1
 
 require (
+	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/delucks/go-subsonic v0.0.0-20220915164742-2744002c4be5 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/glebarez/go-sqlite v1.21.2 // indirect
 	github.com/glebarez/sqlite v1.10.0 // indirect
@@ -17,11 +20,14 @@ require (
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
@@ -33,6 +39,8 @@ require (
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/vbauerster/mpb v3.4.0+incompatible // indirect
+	github.com/vbauerster/mpb/v8 v8.6.2 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect

go.sum

@@ -38,6 +38,10 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
+github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -61,6 +65,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
@@ -151,8 +157,13 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
@@ -165,6 +176,9 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ=
@@ -197,6 +211,10 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/vbauerster/mpb v3.4.0+incompatible h1:mfiiYw87ARaeRW6x5gWwYRUawxaW1tLAD8IceomUCNw=
+github.com/vbauerster/mpb v3.4.0+incompatible/go.mod h1:zAHG26FUhVKETRu+MWqYXcI70POlC6N8up9p1dID7SU=
+github.com/vbauerster/mpb/v8 v8.6.2 h1:9EhnJGQRtvgDVCychJgR96EDCOqgg2NsMuk5JUcX4DA=
+github.com/vbauerster/mpb/v8 v8.6.2/go.mod h1:oVJ7T+dib99kZ/VBjoBaC8aPXiSAihnzuKmotuihyFo=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -353,6 +371,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

models/interfaces.go

@@ -30,33 +30,54 @@ import (
 // A listen service backend.
 // All listen services must implement this interface.
 type Backend interface {
+	// Initialize the backend from a config.
 	FromConfig(config *viper.Viper) Backend
 }
 
+type ImportBackend interface {
+	Backend
+
+	// If the backend needs to setup resources before starting to import,
+	// this can be done here.
+	Init() error
+
+	// The implementation can perform all steps here to finalize the
+	// export/import and free used resources.
+	Finish() error
+}
+
 // Must be implemented by services supporting the export of listens.
 type ListensExport interface {
+	Backend
+
 	// Returns a list of all listens newer then oldestTimestamp.
 	// The returned list of listens is supposed to be ordered by the
 	// Listen.ListenedAt timestamp, with the oldest entry first.
-	ExportListens(oldestTimestamp time.Time, results chan ListensResult)
+	ExportListens(oldestTimestamp time.Time, results chan ListensResult, progress chan Progress)
 }
 
 // Must be implemented by services supporting the import of listens.
 type ListensImport interface {
+	ImportBackend
+
 	// Imports the given list of listens.
-	ImportListens(results chan ListensResult, oldestTimestamp time.Time) (ImportResult, error)
+	ImportListens(export ListensResult, importResult ImportResult, progress chan Progress) (ImportResult, error)
 }
 
 // Must be implemented by services supporting the export of loves.
 type LovesExport interface {
+	Backend
+
 	// Returns a list of all loves newer then oldestTimestamp.
 	// The returned list of listens is supposed to be ordered by the
 	// Love.Created timestamp, with the oldest entry first.
-	ExportLoves(oldestTimestamp time.Time, results chan LovesResult)
+	ExportLoves(oldestTimestamp time.Time, results chan LovesResult, progress chan Progress)
 }
 
 // Must be implemented by services supporting the import of loves.
 type LovesImport interface {
+	ImportBackend
+
 	// Imports the given list of loves.
-	ImportLoves(results chan LovesResult, oldestTimestamp time.Time) (ImportResult, error)
+	ImportLoves(export LovesResult, importResult ImportResult, progress chan Progress) (ImportResult, error)
 }

models/models.go

@@ -105,13 +105,14 @@ func (l LovesList) Swap(i, j int) {
 }
 
 type ListensResult struct {
-	Error   error
+	Listens         ListensList
-	Listens ListensList
+	OldestTimestamp time.Time
+	Error           error
 }
 
 type LovesResult struct {
-	Error error
 	Loves LovesList
+	Error error
 }
 
 type ImportResult struct {
@@ -119,6 +120,9 @@ type ImportResult struct {
 	ImportCount   int
 	LastTimestamp time.Time
 	ImportErrors  []string
+
+	// Error is only set if an unrecoverable import error occurred
+	Error error
 }
 
 // Sets LastTimestamp to newTime, if newTime is newer than LastTimestamp
@@ -127,3 +131,27 @@ func (i *ImportResult) UpdateTimestamp(newTime time.Time) {
 		i.LastTimestamp = newTime
 	}
 }
+
+func (i *ImportResult) Update(from ImportResult) {
+	i.TotalCount = from.TotalCount
+	i.ImportCount += from.ImportCount
+	i.UpdateTimestamp(from.LastTimestamp)
+}
+
+type Progress struct {
+	Total     int64
+	Elapsed   int64
+	Completed bool
+}
+
+func (p Progress) FromImportResult(result ImportResult) Progress {
+	p.Total = int64(result.TotalCount)
+	p.Elapsed = int64(result.ImportCount)
+	return p
+}
+
+func (p Progress) Complete() Progress {
+	p.Total = p.Elapsed
+	p.Completed = true
+	return p
+}

models/models_test.go

@@ -77,6 +77,23 @@ func TestLovesListSort(t *testing.T) {
 	assert.Equal(t, love3, list[1])
 }
 
+func TestImportResultUpdate(t *testing.T) {
+	result := models.ImportResult{
+		TotalCount:    100,
+		ImportCount:   20,
+		LastTimestamp: time.Now(),
+	}
+	newResult := models.ImportResult{
+		TotalCount:    120,
+		ImportCount:   50,
+		LastTimestamp: time.Now().Add(1 * time.Hour),
+	}
+	result.Update(newResult)
+	assert.Equal(t, 120, result.TotalCount)
+	assert.Equal(t, 70, result.ImportCount)
+	assert.Equal(t, newResult.LastTimestamp, result.LastTimestamp)
+}
+
 func TestImportResultUpdateTimestamp(t *testing.T) {
 	timestamp := time.Now()
 	i := models.ImportResult{LastTimestamp: timestamp}