From 6e330daf06ed390b7c61dfa72cd9c8e50f9f23a4 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 16 Nov 2023 00:45:00 +0100 Subject: [PATCH] Implemented progressbar for export/import --- backends/backends.go | 10 +- backends/backends_test.go | 45 ++++++- backends/dump/dump.go | 52 +++----- backends/funkwhale/funkwhale.go | 28 +++- backends/funkwhale/funkwhale_test.go | 4 +- backends/jspf/jspf.go | 42 +++--- backends/jspf/jspf_test.go | 4 +- backends/listenbrainz/listenbrainz.go | 144 +++++++++++---------- backends/listenbrainz/listenbrainz_test.go | 4 +- backends/maloja/maloja.go | 65 +++++----- backends/maloja/maloja_test.go | 4 +- backends/process.go | 107 +++++++++++++++ backends/scrobblerlog/scrobblerlog.go | 70 +++++----- backends/scrobblerlog/scrobblerlog_test.go | 4 +- backends/subsonic/subsonic.go | 8 +- backends/subsonic/subsonic_test.go | 4 +- cmd/listens.go | 21 ++- cmd/loves.go | 22 +++- cmd/progress.go | 84 ++++++++++++ go.mod | 8 ++ go.sum | 19 +++ models/interfaces.go | 29 ++++- models/models.go | 34 ++++- models/models_test.go | 17 +++ 24 files changed, 590 insertions(+), 239 deletions(-) create mode 100644 backends/process.go create mode 100644 cmd/progress.go diff --git a/backends/backends.go b/backends/backends.go index 9f59ae2..99ad3f9 100644 --- a/backends/backends.go +++ b/backends/backends.go @@ -52,7 +52,7 @@ func ResolveBackend[T interface{}](config *viper.Viper) (T, error) { if err != nil { return result, err } - implements, interfaceName := implementsInterface[T](backend) + implements, interfaceName := ImplementsInterface[T](&backend) if implements { result = backend.(T) } else { @@ -91,14 +91,14 @@ func resolveBackend(config *viper.Viper) (string, models.Backend, error) { backendName := config.GetString("backend") backendType := knownBackends[backendName] if backendType == nil { - return backendName, nil, errors.New(fmt.Sprintf("Unknown backend %s", backendName)) + return backendName, nil, fmt.Errorf("Unknown backend %s", backendName) } return backendName, backendType().FromConfig(config), nil } -func implementsInterface[T interface{}](backend models.Backend) (bool, string) { +func ImplementsInterface[T interface{}](backend *models.Backend) (bool, string) { expectedInterface := reflect.TypeOf((*T)(nil)).Elem() - implements := backend != nil && reflect.TypeOf(backend).Implements(expectedInterface) + implements := backend != nil && reflect.TypeOf(*backend).Implements(expectedInterface) return implements, expectedInterface.Name() } @@ -133,7 +133,7 @@ func getImportCapabilities(backend models.Backend) []Capability { } func checkCapability[T interface{}](backend models.Backend, suffix string) (string, bool) { - implements, name := implementsInterface[T](backend) + implements, name := ImplementsInterface[T](&backend) if implements { cap, found := strings.CutSuffix(strings.ToLower(name), suffix) if found { diff --git a/backends/backends_test.go b/backends/backends_test.go index cffb5b7..06d79d4 100644 --- a/backends/backends_test.go +++ b/backends/backends_test.go @@ -23,12 +23,19 @@ THE SOFTWARE. package backends_test import ( + "reflect" "testing" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "go.uploadedlobster.com/scotty/backends" "go.uploadedlobster.com/scotty/backends/dump" + "go.uploadedlobster.com/scotty/backends/funkwhale" + "go.uploadedlobster.com/scotty/backends/jspf" + "go.uploadedlobster.com/scotty/backends/listenbrainz" + "go.uploadedlobster.com/scotty/backends/maloja" + "go.uploadedlobster.com/scotty/backends/scrobblerlog" + "go.uploadedlobster.com/scotty/backends/subsonic" "go.uploadedlobster.com/scotty/models" ) @@ -37,7 +44,7 @@ func TestResolveBackend(t *testing.T) { config.Set("backend", "dump") backend, err := backends.ResolveBackend[models.ListensImport](config) assert.NoError(t, err) - assert.IsType(t, dump.DumpBackend{}, backend) + assert.IsType(t, &dump.DumpBackend{}, backend) } func TestResolveBackendUnknown(t *testing.T) { @@ -69,3 +76,39 @@ func TestGetBackends(t *testing.T) { // If we got here the "dump" backend was not included t.Errorf("GetBackends() did not return expected bacend \"dump\"") } + +func TestImplementsInterfaces(t *testing.T) { + expectInterface[models.ListensImport](t, &dump.DumpBackend{}) + expectInterface[models.LovesImport](t, &dump.DumpBackend{}) + + expectInterface[models.ListensExport](t, &funkwhale.FunkwhaleApiBackend{}) + // expectInterface[models.ListensImport](t, &funkwhale.FunkwhaleApiBackend{}) + expectInterface[models.LovesExport](t, &funkwhale.FunkwhaleApiBackend{}) + // expectInterface[models.LovesImport](t, &funkwhale.FunkwhaleApiBackend{}) + + // expectInterface[models.ListensExport](t, &jspf.JspfBackend{}) + // expectInterface[models.ListensImport](t, &jspf.JspfBackend{}) + // expectInterface[models.LovesExport](t, &jspf.JspfBackend{}) + expectInterface[models.LovesImport](t, &jspf.JspfBackend{}) + + expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{}) + // expectInterface[models.ListensImport](t, &listenbrainz.ListenBrainzApiBackend{}) + expectInterface[models.LovesExport](t, &listenbrainz.ListenBrainzApiBackend{}) + expectInterface[models.LovesImport](t, &listenbrainz.ListenBrainzApiBackend{}) + + expectInterface[models.ListensExport](t, &maloja.MalojaApiBackend{}) + expectInterface[models.ListensImport](t, &maloja.MalojaApiBackend{}) + + expectInterface[models.ListensExport](t, &scrobblerlog.ScrobblerLogBackend{}) + expectInterface[models.ListensImport](t, &scrobblerlog.ScrobblerLogBackend{}) + + expectInterface[models.LovesExport](t, &subsonic.SubsonicApiBackend{}) + // expectInterface[models.LovesImport](t, &subsonic.SubsonicApiBackend{}) +} + +func expectInterface[T interface{}](t *testing.T, backend models.Backend) { + ok, name := backends.ImplementsInterface[T](&backend) + if !ok { + t.Errorf("%v expected to implement %v", reflect.TypeOf(backend).Name(), name) + } +} diff --git a/backends/dump/dump.go b/backends/dump/dump.go index feb8c8d..a946c6e 100644 --- a/backends/dump/dump.go +++ b/backends/dump/dump.go @@ -22,55 +22,39 @@ THE SOFTWARE. package dump import ( - "fmt" - "time" - "github.com/spf13/viper" "go.uploadedlobster.com/scotty/models" ) type DumpBackend struct{} -func (b DumpBackend) FromConfig(config *viper.Viper) models.Backend { +func (b *DumpBackend) FromConfig(config *viper.Viper) models.Backend { return b } -func (b DumpBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) { - importResult := models.ImportResult{ - LastTimestamp: oldestTimestamp, - } - for result := range results { - if result.Error != nil { - return importResult, result.Error - } +func (b *DumpBackend) Init() error { return nil } +func (b *DumpBackend) Finish() error { return nil } - importResult.TotalCount += len(result.Listens) - for _, listen := range result.Listens { - importResult.UpdateTimestamp(listen.ListenedAt) - importResult.ImportCount += 1 - fmt.Printf("🎶 %v: \"%v\" by %v (%v)\n", - listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMbid) - } +func (b *DumpBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { + for _, listen := range export.Listens { + importResult.UpdateTimestamp(listen.ListenedAt) + importResult.ImportCount += 1 + progress <- models.Progress{}.FromImportResult(importResult) + // fmt.Printf("🎶 %v: \"%v\" by %v (%v)\n", + // listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMbid) } + return importResult, nil } -func (b DumpBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) { - importResult := models.ImportResult{ - LastTimestamp: oldestTimestamp, +func (b *DumpBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { + for _, love := range export.Loves { + importResult.UpdateTimestamp(love.Created) + importResult.ImportCount += 1 + progress <- models.Progress{}.FromImportResult(importResult) + // fmt.Printf("❤️ %v: \"%v\" by %v (%v)\n", + // love.Created, love.TrackName, love.ArtistName(), love.RecordingMbid) } - for result := range results { - if result.Error != nil { - return importResult, result.Error - } - importResult.TotalCount += len(result.Loves) - for _, love := range result.Loves { - importResult.UpdateTimestamp(love.Created) - importResult.ImportCount += 1 - fmt.Printf("❤️ %v: \"%v\" by %v (%v)\n", - love.Created, love.TrackName, love.ArtistName(), love.RecordingMbid) - } - } return importResult, nil } diff --git a/backends/funkwhale/funkwhale.go b/backends/funkwhale/funkwhale.go index 613b076..1fb8baa 100644 --- a/backends/funkwhale/funkwhale.go +++ b/backends/funkwhale/funkwhale.go @@ -36,7 +36,7 @@ type FunkwhaleApiBackend struct { username string } -func (b FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend { +func (b *FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend { b.client = NewClient( config.GetString("server-url"), config.GetString("token"), @@ -45,19 +45,22 @@ func (b FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend { return b } -func (b FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) { +func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { page := 1 perPage := MaxItemsPerGet + defer close(results) + defer close(progress) + // We need to gather the full list of listens in order to sort them - listens := make(models.ListensList, 0, 2*MaxItemsPerGet) + listens := make(models.ListensList, 0, 2*perPage) + p := models.Progress{Total: int64(perPage)} out: for { result, err := b.client.GetHistoryListenings(b.username, page, perPage) if err != nil { results <- models.ListensResult{Error: err} - close(results) } count := len(result.Results) @@ -68,6 +71,7 @@ out: for _, fwListen := range result.Results { listen := fwListen.ToListen() if listen.ListenedAt.Unix() > oldestTimestamp.Unix() { + p.Elapsed += 1 listens = append(listens, listen) } else { break out @@ -76,25 +80,31 @@ out: if result.Next == "" { // No further results + p.Total = p.Elapsed + p.Total -= int64(perPage - count) break out } + p.Total += int64(perPage) + progress <- p page += 1 } sort.Sort(listens) + progress <- p.Complete() results <- models.ListensResult{Listens: listens} - close(results) } -func (b FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) { +func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { page := 1 perPage := MaxItemsPerGet defer close(results) + defer close(progress) // We need to gather the full list of listens in order to sort them - loves := make(models.LovesList, 0, 2*MaxItemsPerGet) + loves := make(models.LovesList, 0, 2*perPage) + p := models.Progress{Total: int64(perPage)} out: for { @@ -112,6 +122,7 @@ out: for _, favorite := range result.Results { love := favorite.ToLove() if love.Created.Unix() > oldestTimestamp.Unix() { + p.Elapsed += 1 loves = append(loves, love) } else { break out @@ -123,10 +134,13 @@ out: break out } + p.Total += int64(perPage) + progress <- p page += 1 } sort.Sort(loves) + progress <- p.Complete() results <- models.LovesResult{Loves: loves} } diff --git a/backends/funkwhale/funkwhale_test.go b/backends/funkwhale/funkwhale_test.go index 6b6520e..b151251 100644 --- a/backends/funkwhale/funkwhale_test.go +++ b/backends/funkwhale/funkwhale_test.go @@ -35,8 +35,8 @@ import ( func TestFromConfig(t *testing.T) { config := viper.New() config.Set("token", "thetoken") - backend := funkwhale.FunkwhaleApiBackend{}.FromConfig(config) - assert.IsType(t, funkwhale.FunkwhaleApiBackend{}, backend) + backend := (&funkwhale.FunkwhaleApiBackend{}).FromConfig(config) + assert.IsType(t, &funkwhale.FunkwhaleApiBackend{}, backend) } func TestFunkwhaleListeningToListen(t *testing.T) { diff --git a/backends/jspf/jspf.go b/backends/jspf/jspf.go index d91b8b1..6bdfd8a 100644 --- a/backends/jspf/jspf.go +++ b/backends/jspf/jspf.go @@ -36,42 +36,34 @@ type JspfBackend struct { title string creator string identifier string + tracks []Track } -func (b JspfBackend) FromConfig(config *viper.Viper) models.Backend { +func (b *JspfBackend) FromConfig(config *viper.Viper) models.Backend { b.filePath = config.GetString("file-path") b.title = config.GetString("title") b.creator = config.GetString("username") b.identifier = config.GetString("identifier") + b.tracks = make([]Track, 0) return b } -func (b JspfBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) { - importResult := models.ImportResult{ - LastTimestamp: oldestTimestamp, +func (b *JspfBackend) Init() error { return nil } +func (b *JspfBackend) Finish() error { + err := b.writeJspf(b.tracks) + return err +} + +func (b *JspfBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { + for _, love := range export.Loves { + track := loveToTrack(love) + b.tracks = append(b.tracks, track) + importResult.ImportCount += 1 + importResult.UpdateTimestamp(love.Created) } - tracks := make([]Track, 0, importResult.TotalCount) - for result := range results { - if result.Error != nil { - return importResult, result.Error - } - - importResult.TotalCount += len(result.Loves) - for _, love := range result.Loves { - track := loveToTrack(love) - tracks = append(tracks, track) - oldestTimestamp = love.Created - importResult.ImportCount += 1 - } - } - - err := b.writeJspf(tracks) - if err != nil { - importResult.UpdateTimestamp(oldestTimestamp) - importResult.ImportCount = len(tracks) - } - return importResult, err + progress <- models.Progress{}.FromImportResult(importResult) + return importResult, nil } func loveToTrack(love models.Love) Track { diff --git a/backends/jspf/jspf_test.go b/backends/jspf/jspf_test.go index 2f1f31f..4d13820 100644 --- a/backends/jspf/jspf_test.go +++ b/backends/jspf/jspf_test.go @@ -36,6 +36,6 @@ func TestFromConfig(t *testing.T) { config.Set("title", "My Playlist") config.Set("username", "outsidecontext") config.Set("identifier", "http://example.com/playlist1") - backend := scrobblerlog.ScrobblerLogBackend{}.FromConfig(config) - assert.IsType(t, scrobblerlog.ScrobblerLogBackend{}, backend) + backend := (&scrobblerlog.ScrobblerLogBackend{}).FromConfig(config) + assert.IsType(t, &scrobblerlog.ScrobblerLogBackend{}, backend) } diff --git a/backends/listenbrainz/listenbrainz.go b/backends/listenbrainz/listenbrainz.go index d18f4af..6b1132c 100644 --- a/backends/listenbrainz/listenbrainz.go +++ b/backends/listenbrainz/listenbrainz.go @@ -31,25 +31,34 @@ import ( ) type ListenBrainzApiBackend struct { - client Client - username string + client Client + username string + existingMbids map[string]bool } -func (b ListenBrainzApiBackend) FromConfig(config *viper.Viper) models.Backend { +func (b *ListenBrainzApiBackend) FromConfig(config *viper.Viper) models.Backend { b.client = NewClient(config.GetString("token")) b.client.MaxResults = MaxItemsPerGet b.username = config.GetString("username") return b } -func (b ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) { - maxTime := time.Now() +func (b *ListenBrainzApiBackend) Init() error { return nil } +func (b *ListenBrainzApiBackend) Finish() error { return nil } + +func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { + startTime := time.Now() + maxTime := startTime minTime := time.Unix(0, 0) + totalDuration := startTime.Sub(oldestTimestamp) + defer close(results) + defer close(progress) // FIXME: Optimize by fetching the listens in reverse listen time order listens := make(models.ListensList, 0, 2*MaxItemsPerGet) + p := models.Progress{Total: int64(totalDuration.Seconds())} out: for { @@ -66,6 +75,7 @@ out: // Set maxTime to the oldest returned listen maxTime = time.Unix(result.Payload.Listens[count-1].ListenedAt, 0) + remainingTime := maxTime.Sub(oldestTimestamp) for _, listen := range result.Payload.Listens { if listen.ListenedAt > oldestTimestamp.Unix() { @@ -73,19 +83,26 @@ out: } else { // result contains listens older then oldestTimestamp, // we can stop requesting more + p.Total = int64(startTime.Sub(time.Unix(listen.ListenedAt, 0)).Seconds()) break out } } + + p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) + progress <- p } sort.Sort(listens) - results <- models.ListensResult{Listens: listens} + progress <- p.Complete() + results <- models.ListensResult{Listens: listens, OldestTimestamp: oldestTimestamp} } -func (b ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) { +func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { offset := 0 defer close(results) + defer close(progress) loves := make(models.LovesList, 0, 2*MaxItemsPerGet) + p := models.Progress{} out: for { @@ -104,84 +121,77 @@ out: love := feedback.ToLove() if love.Created.Unix() > oldestTimestamp.Unix() { loves = append(loves, love) + p.Elapsed += 1 + progress <- p } else { break out } } + p.Total = int64(result.TotalCount) + p.Elapsed += int64(count) + offset += MaxItemsPerGet } sort.Sort(loves) + progress <- p.Complete() results <- models.LovesResult{Loves: loves} } -func (b ListenBrainzApiBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) { - importResult := models.ImportResult{ - LastTimestamp: oldestTimestamp, - ImportErrors: make([]string, 0), - } - - existingLovesChan := make(chan models.LovesResult) - go b.ExportLoves(time.Unix(0, 0), existingLovesChan) - existingLoves := <-existingLovesChan - if existingLoves.Error != nil { - results <- models.LovesResult{Error: existingLoves.Error} - close(results) - } - - existingMbids := make(map[string]bool, len(existingLoves.Loves)) - for _, love := range existingLoves.Loves { - existingMbids[string(love.RecordingMbid)] = true - } - - for result := range results { - if result.Error != nil { - return importResult, result.Error +func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { + if len(b.existingMbids) == 0 { + existingLovesChan := make(chan models.LovesResult) + go b.ExportLoves(time.Unix(0, 0), existingLovesChan, progress) + existingLoves := <-existingLovesChan + if existingLoves.Error != nil { + return importResult, existingLoves.Error } - importResult.TotalCount += len(result.Loves) + // TODO: Store MBIDs directly + b.existingMbids = make(map[string]bool, len(existingLoves.Loves)) + for _, love := range existingLoves.Loves { + b.existingMbids[string(love.RecordingMbid)] = true + } + } - for _, love := range result.Loves { - if love.Created.Unix() <= oldestTimestamp.Unix() { - continue - } + for _, love := range export.Loves { + recordingMbid := string(love.RecordingMbid) - recordingMbid := string(love.RecordingMbid) - - if recordingMbid == "" { - lookup, err := b.client.Lookup(love.TrackName, love.ArtistName()) - if err == nil { - recordingMbid = lookup.RecordingMbid - } - } - - if recordingMbid != "" { - ok := false - errMsg := "" - if existingMbids[recordingMbid] { - ok = true - } else { - resp, err := b.client.SendFeedback(Feedback{ - RecordingMbid: recordingMbid, - Score: 1, - }) - ok = err == nil && resp.Status == "ok" - if err != nil { - errMsg = err.Error() - } - } - - if ok { - importResult.UpdateTimestamp(love.Created) - importResult.ImportCount += 1 - } else { - msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v", - love.TrackName, love.ArtistName(), errMsg) - importResult.ImportErrors = append(importResult.ImportErrors, msg) - } + if recordingMbid == "" { + lookup, err := b.client.Lookup(love.TrackName, love.ArtistName()) + if err == nil { + recordingMbid = lookup.RecordingMbid } } + + if recordingMbid != "" { + ok := false + errMsg := "" + if b.existingMbids[recordingMbid] { + ok = true + } else { + resp, err := b.client.SendFeedback(Feedback{ + RecordingMbid: recordingMbid, + Score: 1, + }) + ok = err == nil && resp.Status == "ok" + if err != nil { + errMsg = err.Error() + } + } + + if ok { + importResult.UpdateTimestamp(love.Created) + importResult.ImportCount += 1 + } else { + msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v", + love.TrackName, love.ArtistName(), errMsg) + importResult.ImportErrors = append(importResult.ImportErrors, msg) + } + } + + progress <- models.Progress{}.FromImportResult(importResult) } return importResult, nil diff --git a/backends/listenbrainz/listenbrainz_test.go b/backends/listenbrainz/listenbrainz_test.go index 66c3ab2..6253fed 100644 --- a/backends/listenbrainz/listenbrainz_test.go +++ b/backends/listenbrainz/listenbrainz_test.go @@ -35,8 +35,8 @@ import ( func TestFromConfig(t *testing.T) { config := viper.New() config.Set("token", "thetoken") - backend := listenbrainz.ListenBrainzApiBackend{}.FromConfig(config) - assert.IsType(t, listenbrainz.ListenBrainzApiBackend{}, backend) + backend := (&listenbrainz.ListenBrainzApiBackend{}).FromConfig(config) + assert.IsType(t, &listenbrainz.ListenBrainzApiBackend{}, backend) } func TestListenBrainzListenToListen(t *testing.T) { diff --git a/backends/maloja/maloja.go b/backends/maloja/maloja.go index 42864a9..72b82ff 100644 --- a/backends/maloja/maloja.go +++ b/backends/maloja/maloja.go @@ -36,7 +36,7 @@ type MalojaApiBackend struct { nofix bool } -func (b MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend { +func (b *MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend { b.client = NewClient( config.GetString("server-url"), config.GetString("token"), @@ -45,14 +45,19 @@ func (b MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend { return b } -func (b MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) { +func (b *MalojaApiBackend) Init() error { return nil } +func (b *MalojaApiBackend) Finish() error { return nil } + +func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { page := 0 perPage := MaxItemsPerGet defer close(results) + defer close(progress) // We need to gather the full list of listens in order to sort them listens := make(models.ListensList, 0, 2*perPage) + p := models.Progress{Total: int64(perPage)} out: for { @@ -69,55 +74,45 @@ out: for _, scrobble := range result.List { if scrobble.ListenedAt > oldestTimestamp.Unix() { + p.Elapsed += 1 listens = append(listens, scrobble.ToListen()) } else { break out } } + p.Total += int64(perPage) + progress <- p page += 1 } sort.Sort(listens) + progress <- p.Complete() results <- models.ListensResult{Listens: listens} } -func (b MalojaApiBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) { - importResult := models.ImportResult{ - LastTimestamp: oldestTimestamp, - } - - for result := range results { - if result.Error != nil { - return importResult, result.Error +func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { + for _, listen := range export.Listens { + scrobble := NewScrobble{ + Title: listen.TrackName, + Artists: listen.ArtistNames, + Album: listen.ReleaseName, + Duration: int64(listen.PlaybackDuration.Seconds()), + Length: int64(listen.Duration.Seconds()), + Time: listen.ListenedAt.Unix(), + Nofix: b.nofix, } - importResult.TotalCount += len(result.Listens) - for _, listen := range result.Listens { - if listen.ListenedAt.Unix() <= oldestTimestamp.Unix() { - break - } - - scrobble := NewScrobble{ - Title: listen.TrackName, - Artists: listen.ArtistNames, - Album: listen.ReleaseName, - Duration: int64(listen.PlaybackDuration.Seconds()), - Length: int64(listen.Duration.Seconds()), - Time: listen.ListenedAt.Unix(), - Nofix: b.nofix, - } - - resp, err := b.client.NewScrobble(scrobble) - if err != nil { - return importResult, err - } else if resp.Status != "success" { - return importResult, errors.New(resp.Error.Description) - } - - importResult.UpdateTimestamp(listen.ListenedAt) - importResult.ImportCount += 1 + resp, err := b.client.NewScrobble(scrobble) + if err != nil { + return importResult, err + } else if resp.Status != "success" { + return importResult, errors.New(resp.Error.Description) } + + importResult.UpdateTimestamp(listen.ListenedAt) + importResult.ImportCount += 1 + progress <- models.Progress{}.FromImportResult(importResult) } return importResult, nil diff --git a/backends/maloja/maloja_test.go b/backends/maloja/maloja_test.go index 3860e94..c2dfeac 100644 --- a/backends/maloja/maloja_test.go +++ b/backends/maloja/maloja_test.go @@ -33,8 +33,8 @@ import ( func TestFromConfig(t *testing.T) { config := viper.New() config.Set("token", "thetoken") - backend := maloja.MalojaApiBackend{}.FromConfig(config) - assert.IsType(t, maloja.MalojaApiBackend{}, backend) + backend := (&maloja.MalojaApiBackend{}).FromConfig(config) + assert.IsType(t, &maloja.MalojaApiBackend{}, backend) } func TestScrobbleToListen(t *testing.T) { diff --git a/backends/process.go b/backends/process.go new file mode 100644 index 0000000..d16a867 --- /dev/null +++ b/backends/process.go @@ -0,0 +1,107 @@ +/* +Copyright © 2023 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package backends + +import "go.uploadedlobster.com/scotty/models" + +func ProcessListensImports(importer models.ListensImport, results chan models.ListensResult, out chan models.ImportResult, progress chan models.Progress) { + defer close(out) + defer close(progress) + result := models.ImportResult{} + + err := importer.Init() + if err != nil { + handleError(result, err, out, progress) + return + } + + for exportResult := range results { + if exportResult.Error != nil { + handleError(result, exportResult.Error, out, progress) + return + } + + result.TotalCount += len(exportResult.Listens) + importResult, err := importer.ImportListens(exportResult, result, progress) + if err != nil { + handleError(importResult, err, out, progress) + return + } + + result.Update(importResult) + progress <- models.Progress{}.FromImportResult(result) + } + + err = importer.Finish() + if err != nil { + handleError(result, err, out, progress) + return + } + + progress <- models.Progress{}.FromImportResult(result).Complete() + out <- result +} + +func ProcessLovesImports(importer models.LovesImport, results chan models.LovesResult, out chan models.ImportResult, progress chan models.Progress) { + defer close(out) + defer close(progress) + result := models.ImportResult{} + + err := importer.Init() + if err != nil { + handleError(result, err, out, progress) + return + } + + for exportResult := range results { + if exportResult.Error != nil { + handleError(result, exportResult.Error, out, progress) + return + } + + result.TotalCount += len(exportResult.Loves) + importResult, err := importer.ImportLoves(exportResult, result, progress) + if err != nil { + handleError(importResult, err, out, progress) + return + } + + result.Update(importResult) + progress <- models.Progress{}.FromImportResult(result) + } + + err = importer.Finish() + if err != nil { + handleError(result, err, out, progress) + return + } + + progress <- models.Progress{}.FromImportResult(result).Complete() + out <- result +} + +func handleError(result models.ImportResult, err error, out chan models.ImportResult, progress chan models.Progress) { + result.Error = err + progress <- models.Progress{}.FromImportResult(result).Complete() + out <- result +} diff --git a/backends/scrobblerlog/scrobblerlog.go b/backends/scrobblerlog/scrobblerlog.go index e871602..e5c27ea 100644 --- a/backends/scrobblerlog/scrobblerlog.go +++ b/backends/scrobblerlog/scrobblerlog.go @@ -33,16 +33,44 @@ import ( type ScrobblerLogBackend struct { filePath string includeSkipped bool + file *os.File + log ScrobblerLog } -func (b ScrobblerLogBackend) FromConfig(config *viper.Viper) models.Backend { +func (b *ScrobblerLogBackend) FromConfig(config *viper.Viper) models.Backend { b.filePath = config.GetString("file-path") b.includeSkipped = config.GetBool("include-skipped") return b } -func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) { +func (b *ScrobblerLogBackend) Init() error { + file, err := os.Create(b.filePath) + if err != nil { + return err + } + + b.log = ScrobblerLog{ + Timezone: "UNKNOWN", + Client: "Rockbox unknown $Revision$", + } + + err = WriteHeader(file, &b.log) + if err != nil { + file.Close() + return err + } + + b.file = file + return nil +} + +func (b *ScrobblerLogBackend) Finish() error { + return b.file.Close() +} + +func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { defer close(results) + defer close(progress) file, err := os.Open(b.filePath) if err != nil { results <- models.ListensResult{Error: err} @@ -60,45 +88,19 @@ func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results ch listens := log.Listens.NewerThan(oldestTimestamp) sort.Sort(listens) + progress <- models.Progress{Elapsed: int64(len(listens))}.Complete() results <- models.ListensResult{Listens: listens} } -func (b ScrobblerLogBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) { - importResult := models.ImportResult{ - LastTimestamp: oldestTimestamp, - } - - file, err := os.Create(b.filePath) +func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { + lastTimestamp, err := Write(b.file, export.Listens) if err != nil { return importResult, err } - defer file.Close() - - log := ScrobblerLog{ - Timezone: "UNKNOWN", - Client: "Rockbox unknown $Revision$", - } - - err = WriteHeader(file, &log) - if err != nil { - return importResult, err - } - - for result := range results { - if result.Error != nil { - return importResult, result.Error - } - - importResult.TotalCount += len(result.Listens) - lastTimestamp, err := Write(file, result.Listens) - if err != nil { - return importResult, err - } - - importResult.UpdateTimestamp(lastTimestamp) - importResult.ImportCount += len(result.Listens) - } + importResult.UpdateTimestamp(lastTimestamp) + importResult.ImportCount = len(export.Listens) + progress <- models.Progress{}.FromImportResult(importResult) return importResult, nil } diff --git a/backends/scrobblerlog/scrobblerlog_test.go b/backends/scrobblerlog/scrobblerlog_test.go index 0fa0c4d..9c89757 100644 --- a/backends/scrobblerlog/scrobblerlog_test.go +++ b/backends/scrobblerlog/scrobblerlog_test.go @@ -32,6 +32,6 @@ import ( func TestFromConfig(t *testing.T) { config := viper.New() config.Set("token", "thetoken") - backend := scrobblerlog.ScrobblerLogBackend{}.FromConfig(config) - assert.IsType(t, scrobblerlog.ScrobblerLogBackend{}, backend) + backend := (&scrobblerlog.ScrobblerLogBackend{}).FromConfig(config) + assert.IsType(t, &scrobblerlog.ScrobblerLogBackend{}, backend) } diff --git a/backends/subsonic/subsonic.go b/backends/subsonic/subsonic.go index 0a24567..7f7e630 100644 --- a/backends/subsonic/subsonic.go +++ b/backends/subsonic/subsonic.go @@ -36,7 +36,7 @@ type SubsonicApiBackend struct { password string } -func (b SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend { +func (b *SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend { b.client = subsonic.Client{ Client: &http.Client{}, BaseUrl: config.GetString("server-url"), @@ -47,8 +47,9 @@ func (b SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend { return b } -func (b SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) { +func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { defer close(results) + defer close(progress) err := b.client.Authenticate(b.password) if err != nil { results <- models.LovesResult{Error: err} @@ -61,10 +62,11 @@ func (b SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan return } + progress <- models.Progress{Elapsed: int64(len(starred.Song))}.Complete() results <- models.LovesResult{Loves: b.filterSongs(starred.Song, oldestTimestamp)} } -func (b SubsonicApiBackend) filterSongs(songs []*subsonic.Child, oldestTimestamp time.Time) models.LovesList { +func (b *SubsonicApiBackend) filterSongs(songs []*subsonic.Child, oldestTimestamp time.Time) models.LovesList { loves := make(models.LovesList, len(songs)) for i, song := range songs { love := SongToLove(*song, b.client.User) diff --git a/backends/subsonic/subsonic_test.go b/backends/subsonic/subsonic_test.go index 86b410c..d5ea660 100644 --- a/backends/subsonic/subsonic_test.go +++ b/backends/subsonic/subsonic_test.go @@ -35,8 +35,8 @@ func TestFromConfig(t *testing.T) { config := viper.New() config.Set("server-url", "https://subsonic.example.com") config.Set("token", "thetoken") - backend := subsonic.SubsonicApiBackend{}.FromConfig(config) - assert.IsType(t, subsonic.SubsonicApiBackend{}, backend) + backend := (&subsonic.SubsonicApiBackend{}).FromConfig(config) + assert.IsType(t, &subsonic.SubsonicApiBackend{}, backend) } func TestSongToLove(t *testing.T) { diff --git a/cmd/listens.go b/cmd/listens.go index 52075b3..9233587 100644 --- a/cmd/listens.go +++ b/cmd/listens.go @@ -23,6 +23,7 @@ package cmd import ( "fmt" + "sync" "time" "github.com/spf13/cobra" @@ -60,15 +61,25 @@ var listensCmd = &cobra.Command{ } fmt.Printf("From timestamp: %v (%v)\n", timestamp, timestamp.Unix()) + // Prepare progress bars + exportProgress := make(chan models.Progress) + importProgress := make(chan models.Progress) + var wg sync.WaitGroup + progress := progressBar(&wg, exportProgress, importProgress) + // Export from source - listens := make(chan models.ListensResult, 1000) - go exportBackend.ExportListens(timestamp, listens) + listensChan := make(chan models.ListensResult, 1000) + go exportBackend.ExportListens(timestamp, listensChan, exportProgress) // Import into target - result, err := importBackend.ImportListens(listens, timestamp) - if err != nil { + resultChan := make(chan models.ImportResult) + go backends.ProcessListensImports(importBackend, listensChan, resultChan, importProgress) + result := <-resultChan + wg.Wait() + progress.Wait() + if result.Error != nil { fmt.Printf("Import failed, last reported timestamp was %v (%v)\n", result.LastTimestamp, result.LastTimestamp.Unix()) - cobra.CheckErr(err) + cobra.CheckErr(result.Error) } fmt.Printf("Imported %v of %v listens into %v.\n", result.ImportCount, result.TotalCount, targetName) diff --git a/cmd/loves.go b/cmd/loves.go index 0938bf1..74f4030 100644 --- a/cmd/loves.go +++ b/cmd/loves.go @@ -23,6 +23,7 @@ package cmd import ( "fmt" + "sync" "time" "github.com/spf13/cobra" @@ -60,13 +61,26 @@ var lovesCmd = &cobra.Command{ } fmt.Printf("From timestamp: %v (%v)\n", timestamp, timestamp.Unix()) + // Prepare progress bars + exportProgress := make(chan models.Progress) + importProgress := make(chan models.Progress) + var wg sync.WaitGroup + progress := progressBar(&wg, exportProgress, importProgress) + // Export from source - loves := make(chan models.LovesResult, 1000) - go exportBackend.ExportLoves(timestamp, loves) + lovesChan := make(chan models.LovesResult, 1000) + go exportBackend.ExportLoves(timestamp, lovesChan, exportProgress) // Import into target - result, err := importBackend.ImportLoves(loves, timestamp) - cobra.CheckErr(err) + resultChan := make(chan models.ImportResult) + go backends.ProcessLovesImports(importBackend, lovesChan, resultChan, importProgress) + result := <-resultChan + wg.Wait() + progress.Wait() + if result.Error != nil { + fmt.Printf("Import failed, last reported timestamp was %v (%v)\n", result.LastTimestamp, result.LastTimestamp.Unix()) + cobra.CheckErr(result.Error) + } fmt.Printf("Imported %v of %v loves into %v.\n", result.ImportCount, result.TotalCount, targetName) diff --git a/cmd/progress.go b/cmd/progress.go new file mode 100644 index 0000000..12d8e44 --- /dev/null +++ b/cmd/progress.go @@ -0,0 +1,84 @@ +/* +Copyright © 2023 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package cmd + +import ( + "sync" + "time" + + "github.com/fatih/color" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" + "go.uploadedlobster.com/scotty/models" +) + +func progressBar(wg *sync.WaitGroup, exportProgress chan models.Progress, importProgress chan models.Progress) *mpb.Progress { + p := mpb.New( + mpb.WithWaitGroup(wg), + mpb.WithOutput(color.Output), + // mpb.WithWidth(64), + mpb.WithAutoRefresh(), + ) + + exportBar := setupProgressBar(p, "exporting") + importBar := setupProgressBar(p, "importing") + go updateProgressBar(exportBar, wg, exportProgress) + go updateProgressBar(importBar, wg, importProgress) + + return p +} + +func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar { + green := color.New(color.FgGreen).SprintFunc() + return p.New(0, + mpb.BarStyle(), + mpb.PrependDecorators( + decor.Name(" "), + decor.OnComplete( + decor.Spinner(nil, decor.WC{W: 2, C: decor.DidentRight}), + green("✓ "), + ), + decor.Name(name, decor.WCSyncWidthR), + ), + mpb.AppendDecorators( + decor.OnComplete( + decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth}), + "done", + ), + // decor.OnComplete(decor.Percentage(decor.WC{W: 5, C: decor.DSyncWidthR}), "done"), + decor.Name(" "), + ), + ) +} + +func updateProgressBar(bar *mpb.Bar, wg *sync.WaitGroup, progressChan chan models.Progress) { + wg.Add(1) + defer wg.Done() + lastIterTime := time.Now() + for progress := range progressChan { + oldIterTime := lastIterTime + lastIterTime = time.Now() + bar.EwmaSetCurrent(progress.Elapsed, lastIterTime.Sub(oldIterTime)) + bar.SetTotal(progress.Total, progress.Completed) + } +} diff --git a/go.mod b/go.mod index a4dd154..71d0af1 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,12 @@ module go.uploadedlobster.com/scotty go 1.21.1 require ( + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/delucks/go-subsonic v0.0.0-20220915164742-2744002c4be5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/glebarez/sqlite v1.10.0 // indirect @@ -17,11 +20,14 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -33,6 +39,8 @@ require ( github.com/stretchr/objx v0.5.0 // indirect github.com/stretchr/testify v1.8.4 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/vbauerster/mpb v3.4.0+incompatible // indirect + github.com/vbauerster/mpb/v8 v8.6.2 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect diff --git a/go.sum b/go.sum index 6c6e02f..0d411dc 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,10 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -61,6 +65,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= @@ -151,8 +157,13 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= @@ -165,6 +176,9 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ= @@ -197,6 +211,10 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/vbauerster/mpb v3.4.0+incompatible h1:mfiiYw87ARaeRW6x5gWwYRUawxaW1tLAD8IceomUCNw= +github.com/vbauerster/mpb v3.4.0+incompatible/go.mod h1:zAHG26FUhVKETRu+MWqYXcI70POlC6N8up9p1dID7SU= +github.com/vbauerster/mpb/v8 v8.6.2 h1:9EhnJGQRtvgDVCychJgR96EDCOqgg2NsMuk5JUcX4DA= +github.com/vbauerster/mpb/v8 v8.6.2/go.mod h1:oVJ7T+dib99kZ/VBjoBaC8aPXiSAihnzuKmotuihyFo= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -353,6 +371,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/models/interfaces.go b/models/interfaces.go index fe9c4a5..6d2b294 100644 --- a/models/interfaces.go +++ b/models/interfaces.go @@ -30,33 +30,54 @@ import ( // A listen service backend. // All listen services must implement this interface. type Backend interface { + // Initialize the backend from a config. FromConfig(config *viper.Viper) Backend } +type ImportBackend interface { + Backend + + // If the backend needs to setup resources before starting to import, + // this can be done here. + Init() error + + // The implementation can perform all steps here to finalize the + // export/import and free used resources. + Finish() error +} + // Must be implemented by services supporting the export of listens. type ListensExport interface { + Backend + // Returns a list of all listens newer then oldestTimestamp. // The returned list of listens is supposed to be ordered by the // Listen.ListenedAt timestamp, with the oldest entry first. - ExportListens(oldestTimestamp time.Time, results chan ListensResult) + ExportListens(oldestTimestamp time.Time, results chan ListensResult, progress chan Progress) } // Must be implemented by services supporting the import of listens. type ListensImport interface { + ImportBackend + // Imports the given list of listens. - ImportListens(results chan ListensResult, oldestTimestamp time.Time) (ImportResult, error) + ImportListens(export ListensResult, importResult ImportResult, progress chan Progress) (ImportResult, error) } // Must be implemented by services supporting the export of loves. type LovesExport interface { + Backend + // Returns a list of all loves newer then oldestTimestamp. // The returned list of listens is supposed to be ordered by the // Love.Created timestamp, with the oldest entry first. - ExportLoves(oldestTimestamp time.Time, results chan LovesResult) + ExportLoves(oldestTimestamp time.Time, results chan LovesResult, progress chan Progress) } // Must be implemented by services supporting the import of loves. type LovesImport interface { + ImportBackend + // Imports the given list of loves. - ImportLoves(results chan LovesResult, oldestTimestamp time.Time) (ImportResult, error) + ImportLoves(export LovesResult, importResult ImportResult, progress chan Progress) (ImportResult, error) } diff --git a/models/models.go b/models/models.go index 6fc3373..dbafbd9 100644 --- a/models/models.go +++ b/models/models.go @@ -105,13 +105,14 @@ func (l LovesList) Swap(i, j int) { } type ListensResult struct { - Error error - Listens ListensList + Listens ListensList + OldestTimestamp time.Time + Error error } type LovesResult struct { - Error error Loves LovesList + Error error } type ImportResult struct { @@ -119,6 +120,9 @@ type ImportResult struct { ImportCount int LastTimestamp time.Time ImportErrors []string + + // Error is only set if an unrecoverable import error occurred + Error error } // Sets LastTimestamp to newTime, if newTime is newer than LastTimestamp @@ -127,3 +131,27 @@ func (i *ImportResult) UpdateTimestamp(newTime time.Time) { i.LastTimestamp = newTime } } + +func (i *ImportResult) Update(from ImportResult) { + i.TotalCount = from.TotalCount + i.ImportCount += from.ImportCount + i.UpdateTimestamp(from.LastTimestamp) +} + +type Progress struct { + Total int64 + Elapsed int64 + Completed bool +} + +func (p Progress) FromImportResult(result ImportResult) Progress { + p.Total = int64(result.TotalCount) + p.Elapsed = int64(result.ImportCount) + return p +} + +func (p Progress) Complete() Progress { + p.Total = p.Elapsed + p.Completed = true + return p +} diff --git a/models/models_test.go b/models/models_test.go index 2529e83..d065553 100644 --- a/models/models_test.go +++ b/models/models_test.go @@ -77,6 +77,23 @@ func TestLovesListSort(t *testing.T) { assert.Equal(t, love3, list[1]) } +func TestImportResultUpdate(t *testing.T) { + result := models.ImportResult{ + TotalCount: 100, + ImportCount: 20, + LastTimestamp: time.Now(), + } + newResult := models.ImportResult{ + TotalCount: 120, + ImportCount: 50, + LastTimestamp: time.Now().Add(1 * time.Hour), + } + result.Update(newResult) + assert.Equal(t, 120, result.TotalCount) + assert.Equal(t, 70, result.ImportCount) + assert.Equal(t, newResult.LastTimestamp, result.LastTimestamp) +} + func TestImportResultUpdateTimestamp(t *testing.T) { timestamp := time.Now() i := models.ImportResult{LastTimestamp: timestamp}