Run exporter in goroutine

Use channel to pass data from exporter to importer

backends/dump/dump.go

@@ -35,36 +35,50 @@ func (b DumpBackend) FromConfig(config *viper.Viper) models.Backend {
 	return b
 }
 
-func (b DumpBackend) ImportListens(listens []models.Listen, oldestTimestamp time.Time) (models.ImportResult, error) {
+func (b DumpBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) {
-	result := models.ImportResult{
+	importResult := models.ImportResult{
-		TotalCount:    len(listens),
+		TotalCount:    0,
 		ImportCount:   0,
 		LastTimestamp: oldestTimestamp,
 	}
-	for _, listen := range listens {
+	for result := range results {
-		if listen.ListenedAt.Unix() > result.LastTimestamp.Unix() {
+		if result.Error != nil {
-			result.LastTimestamp = listen.ListenedAt
+			return importResult, result.Error
+		}
+
+		importResult.TotalCount += len(result.Listens)
+		for _, listen := range result.Listens {
+			if listen.ListenedAt.Unix() > importResult.LastTimestamp.Unix() {
+				importResult.LastTimestamp = listen.ListenedAt
+			}
+			importResult.ImportCount += 1
+			fmt.Printf("🎶 %v: \"%v\" by %v (%v)\n",
+				listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMbid)
 		}
-		result.ImportCount += 1
-		fmt.Printf("🎶 %v: \"%v\" by %v (%v)\n",
-			listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMbid)
 	}
-	return result, nil
+	return importResult, nil
 }
 
-func (b DumpBackend) ImportLoves(loves []models.Love, oldestTimestamp time.Time) (models.ImportResult, error) {
+func (b DumpBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
-	result := models.ImportResult{
+	importResult := models.ImportResult{
-		TotalCount:    len(loves),
+		TotalCount:    0,
 		ImportCount:   0,
 		LastTimestamp: oldestTimestamp,
 	}
-	for _, love := range loves {
+	for result := range results {
-		if love.Created.Unix() > result.LastTimestamp.Unix() {
+		if result.Error != nil {
-			result.LastTimestamp = love.Created
+			return importResult, result.Error
+		}
+
+		importResult.TotalCount += len(result.Loves)
+		for _, love := range result.Loves {
+			if love.Created.Unix() > importResult.LastTimestamp.Unix() {
+				importResult.LastTimestamp = love.Created
+			}
+			importResult.ImportCount += 1
+			fmt.Printf("❤️ %v: \"%v\" by %v (%v)\n",
+				love.Created, love.TrackName, love.ArtistName(), love.RecordingMbid)
 		}
-		result.ImportCount += 1
-		fmt.Printf("❤️ %v: \"%v\" by %v (%v)\n",
-			love.Created, love.TrackName, love.ArtistName(), love.RecordingMbid)
 	}
-	return result, nil
+	return importResult, nil
 }

backends/subsonic/subsonic.go

@@ -23,6 +23,7 @@ package subsonic
 
 import (
 	"net/http"
+	"sort"
 	"time"
 
 	"github.com/delucks/go-subsonic"
@@ -46,30 +47,37 @@ func (b SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend {
 	return b
 }
 
-func (b SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time) ([]models.Love, error) {
+func (b SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
 	err := b.client.Authenticate(b.password)
 	if err != nil {
-		return nil, err
+		results <- models.LovesResult{Error: err}
+		close(results)
+		return
 	}
 
-	result, err := b.client.GetStarred2(map[string]string{})
+	starred, err := b.client.GetStarred2(map[string]string{})
 	if err != nil {
-		return nil, err
+		results <- models.LovesResult{Error: err}
+		close(results)
+		return
 	}
 
-	loves := make([]models.Love, 0)
+	results <- models.LovesResult{Loves: b.filterSongs(starred.Song, oldestTimestamp)}
-out:
+	close(results)
-	for _, song := range result.Song {
+	return
+}
+
+func (b SubsonicApiBackend) filterSongs(songs []*subsonic.Child, oldestTimestamp time.Time) models.LovesList {
+	loves := make(models.LovesList, len(songs))
+	for i, song := range songs {
 		love := SongToLove(*song, b.client.User)
 		if love.Created.Unix() > oldestTimestamp.Unix() {
-			loves = append(loves, love)
+			loves[i] = love
-		} else {
-			break out
 		}
 	}
 
-	// TODO: Sort by creation date ascending
+	sort.Sort(loves)
-	return loves, nil
+	return loves
 }
 
 func SongToLove(song subsonic.Child, username string) models.Love {

cmd/listens.go

@@ -61,9 +61,8 @@ var listensCmd = &cobra.Command{
 		fmt.Printf("From timestamp: %v (%v)\n", timestamp, timestamp.Unix())
 
 		// Export from source
-		listens, err := exportBackend.ExportListens(timestamp)
+		listens := make(chan models.ListensResult, 1000)
-		cobra.CheckErr(err)
+		go exportBackend.ExportListens(timestamp, listens)
-		fmt.Printf("Loaded %v listens from %v.\n", len(listens), sourceName)
 
 		// Import into target
 		result, err := importBackend.ImportListens(listens, timestamp)

cmd/loves.go

@@ -61,9 +61,8 @@ var lovesCmd = &cobra.Command{
 		fmt.Printf("From timestamp: %v (%v)\n", timestamp, timestamp.Unix())
 
 		// Export from source
-		loves, err := exportBackend.ExportLoves(timestamp)
+		loves := make(chan models.LovesResult, 1000)
-		cobra.CheckErr(err)
+		go exportBackend.ExportLoves(timestamp, loves)
-		fmt.Printf("Loaded %v loves from %v.\n", len(loves), sourceName)
 
 		// Import into target
 		result, err := importBackend.ImportLoves(loves, timestamp)

models/interfaces.go

@@ -38,13 +38,13 @@ type ListensExport interface {
 	// Returns a list of all listens newer then oldestTimestamp.
 	// The returned list of listens is supposed to be ordered by the
 	// Listen.ListenedAt timestamp, with the oldest entry first.
-	ExportListens(oldestTimestamp time.Time) ([]Listen, error)
+	ExportListens(oldestTimestamp time.Time, results chan ListensResult)
 }
 
 // Must be implemented by services supporting the import of listens.
 type ListensImport interface {
 	// Imports the given list of listens.
-	ImportListens(listens []Listen, oldestTimestamp time.Time) (ImportResult, error)
+	ImportListens(results chan ListensResult, oldestTimestamp time.Time) (ImportResult, error)
 }
 
 // Must be implemented by services supporting the export of loves.
@@ -52,13 +52,23 @@ type LovesExport interface {
 	// Returns a list of all loves newer then oldestTimestamp.
 	// The returned list of listens is supposed to be ordered by the
 	// Love.Created timestamp, with the oldest entry first.
-	ExportLoves(oldestTimestamp time.Time) ([]Love, error)
+	ExportLoves(oldestTimestamp time.Time, results chan LovesResult)
 }
 
 // Must be implemented by services supporting the import of loves.
 type LovesImport interface {
 	// Imports the given list of loves.
-	ImportLoves(loves []Love, oldestTimestamp time.Time) (ImportResult, error)
+	ImportLoves(results chan LovesResult, oldestTimestamp time.Time) (ImportResult, error)
+}
+
+type ListensResult struct {
+	Error   error
+	Listens ListensList
+}
+
+type LovesResult struct {
+	Error error
+	Loves LovesList
 }
 
 type ImportResult struct {

models/models.go

@@ -64,3 +64,31 @@ type Love struct {
 	RecordingMbid MBID
 	RecordingMsid MBID
 }
+
+type ListensList []Listen
+
+func (l ListensList) Len() int {
+	return len(l)
+}
+
+func (l ListensList) Less(i, j int) bool {
+	return l[i].ListenedAt.Unix() < l[j].ListenedAt.Unix()
+}
+
+func (l ListensList) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}
+
+type LovesList []Love
+
+func (l LovesList) Len() int {
+	return len(l)
+}
+
+func (l LovesList) Less(i, j int) bool {
+	return l[i].Created.Unix() < l[j].Created.Unix()
+}
+
+func (l LovesList) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}

models/models_test.go

@@ -22,7 +22,9 @@ THE SOFTWARE.
 package models_test
 
 import (
+	"sort"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"go.uploadedlobster.com/scotty/models"
@@ -38,3 +40,25 @@ func TestTrackArtistName(t *testing.T) {
 	}
 	assert.Equal(t, "Foo, Bar, Baz", track.ArtistName())
 }
+
+func TestListensListSort(t *testing.T) {
+	listen1 := models.Listen{ListenedAt: time.Unix(3, 0)}
+	listen2 := models.Listen{ListenedAt: time.Unix(0, 0)}
+	listen3 := models.Listen{ListenedAt: time.Unix(2, 0)}
+	list := models.ListensList{listen1, listen2, listen3}
+	sort.Sort(list)
+	assert.Equal(t, listen1, list[2])
+	assert.Equal(t, listen2, list[0])
+	assert.Equal(t, listen3, list[1])
+}
+
+func TestLovesListSort(t *testing.T) {
+	love1 := models.Love{Created: time.Unix(3, 0)}
+	love2 := models.Love{Created: time.Unix(0, 0)}
+	love3 := models.Love{Created: time.Unix(2, 0)}
+	list := models.LovesList{love1, love2, love3}
+	sort.Sort(list)
+	assert.Equal(t, love1, list[2])
+	assert.Equal(t, love2, list[0])
+	assert.Equal(t, love3, list[1])
+}