Compare commits

...

3 commits

Author SHA1 Message Date
Philipp Wolfer
a87c42059f
Use a WaitGroup to wait for both export and import goroutine to finish 2025-05-05 17:49:44 +02:00
Philipp Wolfer
17cee9cb8b
For import progress show actually processed and total count 2025-05-05 17:39:47 +02:00
Philipp Wolfer
b8e6ccffdb
Initial implementation of unified export/import progress
Both export and import progress get updated over a unified channel.
Most importantly this allows updating the import total from latest
export results.
2025-05-05 11:38:29 +02:00
18 changed files with 395 additions and 198 deletions

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -18,7 +18,6 @@ Scotty. If not, see <https://www.gnu.org/licenses/>.
package backends_test
import (
"reflect"
"testing"
"github.com/spf13/viper"
@ -33,6 +32,7 @@ import (
"go.uploadedlobster.com/scotty/internal/backends/maloja"
"go.uploadedlobster.com/scotty/internal/backends/scrobblerlog"
"go.uploadedlobster.com/scotty/internal/backends/spotify"
"go.uploadedlobster.com/scotty/internal/backends/spotifyhistory"
"go.uploadedlobster.com/scotty/internal/backends/subsonic"
"go.uploadedlobster.com/scotty/internal/config"
"go.uploadedlobster.com/scotty/internal/i18n"
@ -93,9 +93,9 @@ func TestImplementsInterfaces(t *testing.T) {
expectInterface[models.LovesExport](t, &funkwhale.FunkwhaleApiBackend{})
// expectInterface[models.LovesImport](t, &funkwhale.FunkwhaleApiBackend{})
// expectInterface[models.ListensExport](t, &jspf.JSPFBackend{})
expectInterface[models.ListensExport](t, &jspf.JSPFBackend{})
expectInterface[models.ListensImport](t, &jspf.JSPFBackend{})
// expectInterface[models.LovesExport](t, &jspf.JSPFBackend{})
expectInterface[models.LovesExport](t, &jspf.JSPFBackend{})
expectInterface[models.LovesImport](t, &jspf.JSPFBackend{})
// expectInterface[models.ListensExport](t, &lastfm.LastfmApiBackend{})
@ -115,6 +115,8 @@ func TestImplementsInterfaces(t *testing.T) {
expectInterface[models.LovesExport](t, &spotify.SpotifyApiBackend{})
// expectInterface[models.LovesImport](t, &spotify.SpotifyApiBackend{})
expectInterface[models.ListensExport](t, &spotifyhistory.SpotifyHistoryBackend{})
expectInterface[models.ListensExport](t, &scrobblerlog.ScrobblerLogBackend{})
expectInterface[models.ListensImport](t, &scrobblerlog.ScrobblerLogBackend{})
@ -125,6 +127,6 @@ func TestImplementsInterfaces(t *testing.T) {
func expectInterface[T interface{}](t *testing.T, backend models.Backend) {
ok, name := backends.ImplementsInterface[T](&backend)
if !ok {
t.Errorf("%v expected to implement %v", reflect.TypeOf(backend).Name(), name)
t.Errorf("%v expected to implement %v", backend.Name(), name)
}
}

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
Scotty is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
@ -77,7 +77,7 @@ func (b *DeezerApiBackend) OAuth2Setup(token oauth2.TokenSource) error {
return nil
}
func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
// Choose a high offset, we attempt to search the loves backwards starting
// at the oldest one.
offset := math.MaxInt32
@ -88,13 +88,18 @@ func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan
totalDuration := startTime.Sub(oldestTimestamp)
p := models.Progress{Total: int64(totalDuration.Seconds())}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(totalDuration.Seconds()),
},
}
out:
for {
result, err := b.client.UserHistory(offset, perPage)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -102,7 +107,6 @@ out:
// The offset was higher then the actual number of tracks. Adjust the offset
// and continue.
if offset >= result.Total {
p.Total = int64(result.Total)
offset = max(result.Total-perPage, 0)
continue
}
@ -128,7 +132,8 @@ out:
}
remainingTime := startTime.Sub(minTime)
p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
p.Export.TotalItems += len(listens)
p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
progress <- p
results <- models.ListensResult{Items: listens, OldestTimestamp: minTime}
@ -144,23 +149,29 @@ out:
}
results <- models.ListensResult{OldestTimestamp: minTime}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (b *DeezerApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *DeezerApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
// Choose a high offset, we attempt to search the loves backwards starting
// at the oldest one.
offset := math.MaxInt32
perPage := MaxItemsPerGet
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
var totalCount int
out:
for {
result, err := b.client.UserTracks(offset, perPage)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
@ -168,8 +179,8 @@ out:
// The offset was higher then the actual number of tracks. Adjust the offset
// and continue.
if offset >= result.Total {
p.Total = int64(result.Total)
totalCount = result.Total
p.Export.Total = int64(totalCount)
offset = max(result.Total-perPage, 0)
continue
}
@ -186,13 +197,14 @@ out:
loves = append(loves, love)
} else {
totalCount -= 1
break
}
}
sort.Sort(loves)
results <- models.LovesResult{Items: loves, Total: totalCount}
p.Elapsed += int64(count)
p.Export.TotalItems = totalCount
p.Export.Total = int64(totalCount)
p.Export.Elapsed += int64(count)
progress <- p
if offset <= 0 {
@ -206,7 +218,8 @@ out:
}
}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (t Listen) AsListen() models.Listen {

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -36,27 +36,27 @@ func (b *DumpBackend) InitConfig(config *config.ServiceConfig) error {
func (b *DumpBackend) StartImport() error { return nil }
func (b *DumpBackend) FinishImport() error { return nil }
func (b *DumpBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *DumpBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
for _, listen := range export.Items {
importResult.UpdateTimestamp(listen.ListenedAt)
importResult.ImportCount += 1
msg := fmt.Sprintf("🎶 %v: \"%v\" by %v (%v)",
listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMBID)
importResult.Log(models.Info, msg)
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
}
return importResult, nil
}
func (b *DumpBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *DumpBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
for _, love := range export.Items {
importResult.UpdateTimestamp(love.Created)
importResult.ImportCount += 1
msg := fmt.Sprintf("❤️ %v: \"%v\" by %v (%v)",
love.Created, love.TrackName, love.ArtistName(), love.RecordingMBID)
importResult.Log(models.Info, msg)
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
}
return importResult, nil

View file

@ -16,6 +16,7 @@ Scotty. If not, see <https://www.gnu.org/licenses/>.
package backends
import (
"sync"
"time"
"go.uploadedlobster.com/scotty/internal/models"
@ -23,7 +24,7 @@ import (
type ExportProcessor[T models.ListensResult | models.LovesResult] interface {
ExportBackend() models.Backend
Process(oldestTimestamp time.Time, results chan T, progress chan models.Progress)
Process(wg *sync.WaitGroup, oldestTimestamp time.Time, results chan T, progress chan models.TransferProgress)
}
type ListensExportProcessor struct {
@ -34,9 +35,10 @@ func (p ListensExportProcessor) ExportBackend() models.Backend {
return p.Backend
}
func (p ListensExportProcessor) Process(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (p ListensExportProcessor) Process(wg *sync.WaitGroup, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
wg.Add(1)
defer wg.Done()
defer close(results)
defer close(progress)
p.Backend.ExportListens(oldestTimestamp, results, progress)
}
@ -48,8 +50,9 @@ func (p LovesExportProcessor) ExportBackend() models.Backend {
return p.Backend
}
func (p LovesExportProcessor) Process(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (p LovesExportProcessor) Process(wg *sync.WaitGroup, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
wg.Add(1)
defer wg.Done()
defer close(results)
defer close(progress)
p.Backend.ExportLoves(oldestTimestamp, results, progress)
}

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -60,19 +60,26 @@ func (b *FunkwhaleApiBackend) InitConfig(config *config.ServiceConfig) error {
return nil
}
func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
page := 1
perPage := MaxItemsPerGet
// We need to gather the full list of listens in order to sort them
listens := make(models.ListensList, 0, 2*perPage)
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
out:
for {
result, err := b.client.GetHistoryListenings(b.username, page, perPage)
if err != nil {
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
count := len(result.Results)
@ -83,7 +90,7 @@ out:
for _, fwListen := range result.Results {
listen := fwListen.AsListen()
if listen.ListenedAt.After(oldestTimestamp) {
p.Elapsed += 1
p.Export.Elapsed += 1
listens = append(listens, listen)
} else {
break out
@ -92,34 +99,42 @@ out:
if result.Next == "" {
// No further results
p.Total = p.Elapsed
p.Total -= int64(perPage - count)
p.Export.Total = p.Export.Elapsed
p.Export.Total -= int64(perPage - count)
break out
}
p.Total += int64(perPage)
p.Export.TotalItems = len(listens)
p.Export.Total += int64(perPage)
progress <- p
page += 1
}
sort.Sort(listens)
progress <- p.Complete()
p.Export.TotalItems = len(listens)
p.Export.Complete()
progress <- p
results <- models.ListensResult{Items: listens}
}
func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
page := 1
perPage := MaxItemsPerGet
// We need to gather the full list of listens in order to sort them
loves := make(models.LovesList, 0, 2*perPage)
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
out:
for {
result, err := b.client.GetFavoriteTracks(page, perPage)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
@ -132,7 +147,7 @@ out:
for _, favorite := range result.Results {
love := favorite.AsLove()
if love.Created.After(oldestTimestamp) {
p.Elapsed += 1
p.Export.Elapsed += 1
loves = append(loves, love)
} else {
break out
@ -144,13 +159,16 @@ out:
break out
}
p.Total += int64(perPage)
p.Export.TotalItems = len(loves)
p.Export.Total += int64(perPage)
progress <- p
page += 1
}
sort.Sort(loves)
progress <- p.Complete()
p.Export.TotalItems = len(loves)
p.Export.Complete()
progress <- p
results <- models.LovesResult{Items: loves}
}

View file

@ -18,13 +18,15 @@ Scotty. If not, see <https://www.gnu.org/licenses/>.
package backends
import (
"sync"
"go.uploadedlobster.com/scotty/internal/models"
)
type ImportProcessor[T models.ListensResult | models.LovesResult] interface {
ImportBackend() models.ImportBackend
Process(results chan T, out chan models.ImportResult, progress chan models.Progress)
Import(export T, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error)
Process(wg *sync.WaitGroup, results chan T, out chan models.ImportResult, progress chan models.TransferProgress)
Import(export T, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error)
}
type ListensImportProcessor struct {
@ -35,11 +37,11 @@ func (p ListensImportProcessor) ImportBackend() models.ImportBackend {
return p.Backend
}
func (p ListensImportProcessor) Process(results chan models.ListensResult, out chan models.ImportResult, progress chan models.Progress) {
process(p, results, out, progress)
func (p ListensImportProcessor) Process(wg *sync.WaitGroup, results chan models.ListensResult, out chan models.ImportResult, progress chan models.TransferProgress) {
process(wg, p, results, out, progress)
}
func (p ListensImportProcessor) Import(export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (p ListensImportProcessor) Import(export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
if export.Error != nil {
return result, export.Error
}
@ -64,11 +66,11 @@ func (p LovesImportProcessor) ImportBackend() models.ImportBackend {
return p.Backend
}
func (p LovesImportProcessor) Process(results chan models.LovesResult, out chan models.ImportResult, progress chan models.Progress) {
process(p, results, out, progress)
func (p LovesImportProcessor) Process(wg *sync.WaitGroup, results chan models.LovesResult, out chan models.ImportResult, progress chan models.TransferProgress) {
process(wg, p, results, out, progress)
}
func (p LovesImportProcessor) Import(export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (p LovesImportProcessor) Import(export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
if export.Error != nil {
return result, export.Error
}
@ -85,11 +87,12 @@ func (p LovesImportProcessor) Import(export models.LovesResult, result models.Im
return importResult, nil
}
func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](processor P, results chan R, out chan models.ImportResult, progress chan models.Progress) {
func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](wg *sync.WaitGroup, processor P, results chan R, out chan models.ImportResult, progress chan models.TransferProgress) {
wg.Add(1)
defer wg.Done()
defer close(out)
defer close(progress)
result := models.ImportResult{}
p := models.Progress{}
p := models.TransferProgress{}
if err := processor.ImportBackend().StartImport(); err != nil {
out <- handleError(result, err, progress)
@ -104,7 +107,7 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](
out <- handleError(result, err, progress)
return
}
progress <- p.FromImportResult(result)
progress <- p.FromImportResult(result, false)
}
if err := processor.ImportBackend().FinishImport(); err != nil {
@ -112,12 +115,14 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](
return
}
progress <- p.FromImportResult(result).Complete()
progress <- p.FromImportResult(result, true)
out <- result
}
func handleError(result models.ImportResult, err error, progress chan models.Progress) models.ImportResult {
func handleError(result models.ImportResult, err error, progress chan models.TransferProgress) models.ImportResult {
result.Error = err
progress <- models.Progress{}.FromImportResult(result).Abort()
p := models.TransferProgress{}.FromImportResult(result, false)
p.Import.Abort()
progress <- p
return result
}

View file

@ -93,10 +93,15 @@ func (b *JSPFBackend) FinishImport() error {
return b.writeJSPF()
}
func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
err := b.readJSPF()
p := models.TransferProgress{
Export: &models.Progress{},
}
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -109,11 +114,13 @@ func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan mode
}
}
sort.Sort(listens)
progress <- models.Progress{Total: int64(len(listens))}.Complete()
p.Export.Total = int64(len(listens))
p.Export.Complete()
progress <- p
results <- models.ListensResult{Items: listens}
}
func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
for _, listen := range export.Items {
track := listenAsTrack(listen)
b.playlist.Tracks = append(b.playlist.Tracks, track)
@ -121,14 +128,19 @@ func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult mo
importResult.UpdateTimestamp(listen.ListenedAt)
}
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
return importResult, nil
}
func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
err := b.readJSPF()
p := models.TransferProgress{
Export: &models.Progress{},
}
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
@ -141,11 +153,13 @@ func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models
}
}
sort.Sort(loves)
progress <- models.Progress{Total: int64(len(loves))}.Complete()
p.Export.Total = int64(len(loves))
p.Export.Complete()
progress <- p
results <- models.LovesResult{Items: loves}
}
func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
for _, love := range export.Items {
track := loveAsTrack(love)
b.playlist.Tracks = append(b.playlist.Tracks, track)
@ -153,7 +167,7 @@ func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models
importResult.UpdateTimestamp(love.Created)
}
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
return importResult, nil
}

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
Scotty is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
@ -88,13 +88,17 @@ func (b *LastfmApiBackend) OAuth2Setup(token oauth2.TokenSource) error {
return nil
}
func (b *LastfmApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *LastfmApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
page := MaxPage
minTime := oldestTimestamp
perPage := MaxItemsPerGet
// We need to gather the full list of listens in order to sort them
p := models.Progress{Total: int64(page)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(page),
},
}
out:
for page > 0 {
@ -108,7 +112,8 @@ out:
result, err := b.client.User.GetRecentTracks(args)
if err != nil {
results <- models.ListensResult{Error: err}
progress <- p.Abort()
p.Export.Abort()
progress <- p
return
}
@ -127,11 +132,12 @@ out:
timestamp, err := strconv.ParseInt(scrobble.Date.Uts, 10, 64)
if err != nil {
results <- models.ListensResult{Error: err}
progress <- p.Abort()
p.Export.Abort()
progress <- p
break out
}
if timestamp > oldestTimestamp.Unix() {
p.Elapsed += 1
p.Export.Elapsed += 1
listen := models.Listen{
ListenedAt: time.Unix(timestamp, 0),
UserName: b.username,
@ -165,16 +171,18 @@ out:
Total: result.Total,
OldestTimestamp: minTime,
}
p.Total = int64(result.TotalPages)
p.Elapsed = int64(result.TotalPages - page)
p.Export.Total = int64(result.TotalPages)
p.Export.Elapsed = int64(result.TotalPages - page)
p.Export.TotalItems += len(listens)
progress <- p
}
results <- models.ListensResult{OldestTimestamp: minTime}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
total := len(export.Items)
for i := 0; i < total; i += MaxListensPerSubmission {
listens := export.Items[i:min(i+MaxListensPerSubmission, total)]
@ -244,20 +252,24 @@ func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResu
importResult.UpdateTimestamp(listens[count-1].ListenedAt)
importResult.ImportCount += accepted
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
}
return importResult, nil
}
func (b *LastfmApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *LastfmApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
// Choose a high offset, we attempt to search the loves backwards starting
// at the oldest one.
page := 1
perPage := MaxItemsPerGet
loves := make(models.LovesList, 0, 2*MaxItemsPerGet)
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
var totalCount int
out:
@ -268,12 +280,12 @@ out:
"page": page,
})
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
p.Total = int64(result.Total)
count := len(result.Tracks)
if count == 0 {
break out
@ -282,7 +294,8 @@ out:
for _, track := range result.Tracks {
timestamp, err := strconv.ParseInt(track.Date.Uts, 10, 64)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
@ -308,18 +321,21 @@ out:
}
}
p.Elapsed += int64(count)
p.Export.Total += int64(perPage)
p.Export.TotalItems = totalCount
p.Export.Elapsed += int64(count)
progress <- p
page += 1
}
sort.Sort(loves)
p.Export.Complete()
progress <- p
results <- models.LovesResult{Items: loves, Total: totalCount}
progress <- p.Complete()
}
func (b *LastfmApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *LastfmApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
for _, love := range export.Items {
err := b.client.Track.Love(lastfm.P{
"track": love.TrackName,
@ -335,7 +351,7 @@ func (b *LastfmApiBackend) ImportLoves(export models.LovesResult, importResult m
importResult.Log(models.Error, msg)
}
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
}
return importResult, nil

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -72,21 +72,25 @@ func (b *ListenBrainzApiBackend) InitConfig(config *config.ServiceConfig) error
func (b *ListenBrainzApiBackend) StartImport() error { return nil }
func (b *ListenBrainzApiBackend) FinishImport() error { return nil }
func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
startTime := time.Now()
minTime := oldestTimestamp
if minTime.Unix() < 1 {
minTime = time.Unix(1, 0)
}
totalDuration := startTime.Sub(minTime)
p := models.Progress{Total: int64(totalDuration.Seconds())}
totalDuration := startTime.Sub(oldestTimestamp)
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(totalDuration.Seconds()),
},
}
for {
result, err := b.client.GetListens(b.username, time.Now(), minTime)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -96,7 +100,7 @@ func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, result
if minTime.Unix() < result.Payload.OldestListenTimestamp {
minTime = time.Unix(result.Payload.OldestListenTimestamp, 0)
totalDuration = startTime.Sub(minTime)
p.Total = int64(totalDuration.Seconds())
p.Export.Total = int64(totalDuration.Seconds())
continue
} else {
break
@ -119,18 +123,20 @@ func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, result
}
sort.Sort(listens)
p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
p.Export.TotalItems += len(listens)
p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
progress <- p
results <- models.ListensResult{Items: listens, OldestTimestamp: minTime}
}
results <- models.ListensResult{OldestTimestamp: minTime}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
total := len(export.Items)
p := models.Progress{}.FromImportResult(importResult)
p := models.TransferProgress{}.FromImportResult(importResult, false)
for i := 0; i < total; i += MaxListensPerRequest {
listens := export.Items[i:min(i+MaxListensPerRequest, total)]
count := len(listens)
@ -146,7 +152,7 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo
for _, l := range listens {
if b.checkDuplicates {
isDupe, err := b.checkDuplicateListen(l)
p.Elapsed += 1
p.Import.Elapsed += 1
progress <- p
if err != nil {
return importResult, err
@ -186,31 +192,36 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo
importResult.UpdateTimestamp(listens[count-1].ListenedAt)
}
importResult.ImportCount += count
progress <- p.FromImportResult(importResult)
progress <- p.FromImportResult(importResult, false)
}
return importResult, nil
}
func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
exportChan := make(chan models.LovesResult)
p := models.Progress{}
p := models.TransferProgress{
Export: &models.Progress{},
}
go b.exportLoves(oldestTimestamp, exportChan)
for existingLoves := range exportChan {
if existingLoves.Error != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: existingLoves.Error}
return
}
p.Total = int64(existingLoves.Total)
p.Elapsed += int64(existingLoves.Items.Len())
p.Export.TotalItems = existingLoves.Total
p.Export.Total = int64(existingLoves.Total)
p.Export.Elapsed += int64(len(existingLoves.Items))
progress <- p
results <- existingLoves
}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (b *ListenBrainzApiBackend) exportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
@ -260,7 +271,7 @@ out:
}
}
func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
if len(b.existingMBIDs) == 0 {
existingLovesChan := make(chan models.LovesResult)
go b.exportLoves(time.Unix(0, 0), existingLovesChan)
@ -330,7 +341,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importRe
importResult.Log(models.Error, msg)
}
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
}
return importResult, nil

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -63,19 +63,24 @@ func (b *MalojaApiBackend) InitConfig(config *config.ServiceConfig) error {
func (b *MalojaApiBackend) StartImport() error { return nil }
func (b *MalojaApiBackend) FinishImport() error { return nil }
func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
page := 0
perPage := MaxItemsPerGet
// We need to gather the full list of listens in order to sort them
listens := make(models.ListensList, 0, 2*perPage)
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
out:
for {
result, err := b.client.GetScrobbles(page, perPage)
if err != nil {
progress <- p.Complete()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -87,25 +92,27 @@ out:
for _, scrobble := range result.List {
if scrobble.ListenedAt > oldestTimestamp.Unix() {
p.Elapsed += 1
p.Export.Elapsed += 1
listens = append(listens, scrobble.AsListen())
} else {
break out
}
}
p.Total += int64(perPage)
p.Export.TotalItems = len(listens)
p.Export.Total += int64(perPage)
progress <- p
page += 1
}
sort.Sort(listens)
progress <- p.Complete()
p.Export.Complete()
progress <- p
results <- models.ListensResult{Items: listens}
}
func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
p := models.Progress{}.FromImportResult(importResult)
func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
p := models.TransferProgress{}.FromImportResult(importResult, false)
for _, listen := range export.Items {
scrobble := NewScrobble{
Title: listen.TrackName,
@ -126,7 +133,7 @@ func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResu
importResult.UpdateTimestamp(listen.ListenedAt)
importResult.ImportCount += 1
progress <- p.FromImportResult(importResult)
progress <- p.FromImportResult(importResult, false)
}
return importResult, nil

View file

@ -131,10 +131,14 @@ func (b *ScrobblerLogBackend) FinishImport() error {
return b.file.Close()
}
func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
file, err := os.Open(b.filePath)
p := models.TransferProgress{
Export: &models.Progress{},
}
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -143,7 +147,8 @@ func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results c
err = b.log.Parse(file, b.ignoreSkipped)
if err != nil {
progress <- models.Progress{}.Complete()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -157,11 +162,13 @@ func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results c
}
}
sort.Sort(listens)
progress <- models.Progress{Total: int64(len(listens))}.Complete()
p.Export.Total = int64(len(listens))
p.Export.Complete()
progress <- p
results <- models.ListensResult{Items: listens}
}
func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
records := make([]scrobblerlog.Record, len(export.Items))
for i, listen := range export.Items {
records[i] = listenToRecord(listen)
@ -173,7 +180,7 @@ func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importR
importResult.UpdateTimestamp(lastTimestamp)
importResult.ImportCount += len(export.Items)
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
return importResult, nil
}

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -95,18 +95,22 @@ func (b *SpotifyApiBackend) OAuth2Setup(token oauth2.TokenSource) error {
return nil
}
func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
startTime := time.Now()
minTime := oldestTimestamp
totalDuration := startTime.Sub(oldestTimestamp)
p := models.Progress{Total: int64(totalDuration.Seconds())}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(totalDuration.Seconds()),
},
}
for {
result, err := b.client.RecentlyPlayedAfter(minTime, MaxItemsPerGet)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -118,7 +122,8 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha
// Set minTime to the newest returned listen
after, err := strconv.ParseInt(result.Cursors.After, 10, 64)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
} else if after <= minTime.Unix() {
@ -146,22 +151,28 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha
}
sort.Sort(listens)
p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
p.Export.TotalItems += len(listens)
p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
progress <- p
results <- models.ListensResult{Items: listens, OldestTimestamp: minTime}
}
results <- models.ListensResult{OldestTimestamp: minTime}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (b *SpotifyApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *SpotifyApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
// Choose a high offset, we attempt to search the loves backwards starting
// at the oldest one.
offset := math.MaxInt32
perPage := MaxItemsPerGet
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
totalCount := 0
exportCount := 0
@ -169,7 +180,8 @@ out:
for {
result, err := b.client.UserTracks(offset, perPage)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
@ -177,7 +189,7 @@ out:
// The offset was higher then the actual number of tracks. Adjust the offset
// and continue.
if offset >= result.Total {
p.Total = int64(result.Total)
p.Export.Total = int64(result.Total)
totalCount = result.Total
offset = max(result.Total-perPage, 0)
continue
@ -201,7 +213,7 @@ out:
exportCount += len(loves)
sort.Sort(loves)
results <- models.LovesResult{Items: loves, Total: totalCount}
p.Elapsed += int64(count)
p.Export.Elapsed += int64(count)
progress <- p
if offset <= 0 {
@ -216,7 +228,8 @@ out:
}
results <- models.LovesResult{Total: exportCount}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (l Listen) AsListen() models.Listen {

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -72,21 +72,27 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error {
return nil
}
func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob))
p := models.TransferProgress{
Export: &models.Progress{},
}
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
slices.Sort(files)
fileCount := int64(len(files))
p := models.Progress{Total: fileCount}
p.Export.Total = fileCount
for i, filePath := range files {
history, err := readHistoryFile(filePath)
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -97,11 +103,13 @@ func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results
})
sort.Sort(listens)
results <- models.ListensResult{Items: listens}
p.Elapsed = int64(i)
p.Export.Elapsed = int64(i)
p.Export.TotalItems += len(listens)
progress <- p
}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func readHistoryFile(filePath string) (StreamingHistory, error) {

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -63,25 +63,30 @@ func (b *SubsonicApiBackend) InitConfig(config *config.ServiceConfig) error {
return nil
}
func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
err := b.client.Authenticate(b.password)
p := models.TransferProgress{
Export: &models.Progress{},
}
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
starred, err := b.client.GetStarred2(map[string]string{})
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
loves := b.filterSongs(starred.Song, oldestTimestamp)
progress <- models.Progress{
Total: int64(loves.Len()),
}.Complete()
p.Export.Total = int64(len(loves))
p.Export.Complete()
progress <- p
results <- models.LovesResult{Items: loves}
}

View file

@ -28,7 +28,18 @@ import (
"go.uploadedlobster.com/scotty/internal/models"
)
func progressBar(exportProgress chan models.Progress, importProgress chan models.Progress) *mpb.Progress {
type progressBarUpdater struct {
wg *sync.WaitGroup
progress *mpb.Progress
exportBar *mpb.Bar
importBar *mpb.Bar
updateChan chan models.TransferProgress
lastExportUpdate time.Time
totalItems int
importedItems int
}
func setupProgressBars(updateChan chan models.TransferProgress) progressBarUpdater {
wg := &sync.WaitGroup{}
p := mpb.New(
mpb.WithWaitGroup(wg),
@ -37,16 +48,81 @@ func progressBar(exportProgress chan models.Progress, importProgress chan models
mpb.WithAutoRefresh(),
)
exportBar := setupProgressBar(p, i18n.Tr("exporting"))
importBar := setupProgressBar(p, i18n.Tr("importing"))
go updateProgressBar(exportBar, wg, exportProgress)
go updateProgressBar(importBar, wg, importProgress)
u := progressBarUpdater{
wg: wg,
progress: p,
exportBar: initExportProgressBar(p, i18n.Tr("exporting")),
importBar: initImportProgressBar(p, i18n.Tr("importing")),
updateChan: updateChan,
}
return p
go u.update()
return u
}
func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar {
func (u *progressBarUpdater) close() {
close(u.updateChan)
u.progress.Wait()
}
func (u *progressBarUpdater) update() {
u.wg.Add(1)
defer u.wg.Done()
u.lastExportUpdate = time.Now()
for progress := range u.updateChan {
if progress.Export != nil {
u.updateExportProgress(progress.Export)
}
if progress.Import != nil {
if int64(u.totalItems) > progress.Import.Total {
progress.Import.Total = int64(u.totalItems)
}
u.updateImportProgress(progress.Import)
}
}
}
func (u *progressBarUpdater) updateExportProgress(progress *models.Progress) {
bar := u.exportBar
u.totalItems = progress.TotalItems
if progress.Aborted {
bar.Abort(false)
return
}
oldIterTime := u.lastExportUpdate
u.lastExportUpdate = time.Now()
elapsedTime := u.lastExportUpdate.Sub(oldIterTime)
bar.EwmaSetCurrent(progress.Elapsed, elapsedTime)
bar.SetTotal(progress.Total, progress.Completed)
}
func (u *progressBarUpdater) updateImportProgress(progress *models.Progress) {
bar := u.importBar
if progress.Aborted {
bar.Abort(false)
return
}
bar.SetCurrent(progress.Elapsed)
bar.SetTotal(progress.Total, progress.Completed)
}
func initExportProgressBar(p *mpb.Progress, name string) *mpb.Bar {
return initProgressBar(p, name,
decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth}))
}
func initImportProgressBar(p *mpb.Progress, name string) *mpb.Bar {
return initProgressBar(p, name, decor.Counters(0, "%d / %d"))
}
func initProgressBar(p *mpb.Progress, name string, progressDecorator decor.Decorator) *mpb.Bar {
green := color.New(color.FgGreen).SprintFunc()
red := color.New(color.FgHiRed, color.Bold).SprintFunc()
return p.New(0,
mpb.BarStyle(),
mpb.PrependDecorators(
@ -60,8 +136,8 @@ func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar {
mpb.AppendDecorators(
decor.OnComplete(
decor.OnAbort(
decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth}),
i18n.Tr("aborted"),
progressDecorator,
red(i18n.Tr("aborted")),
),
i18n.Tr("done"),
),
@ -69,19 +145,3 @@ func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar {
),
)
}
func updateProgressBar(bar *mpb.Bar, wg *sync.WaitGroup, progressChan chan models.Progress) {
wg.Add(1)
defer wg.Done()
lastIterTime := time.Now()
for progress := range progressChan {
if progress.Aborted {
bar.Abort(false)
return
}
oldIterTime := lastIterTime
lastIterTime = time.Now()
bar.EwmaSetCurrent(progress.Elapsed, lastIterTime.Sub(oldIterTime))
bar.SetTotal(progress.Total, progress.Completed)
}
}

View file

@ -19,6 +19,7 @@ import (
"errors"
"fmt"
"strconv"
"sync"
"time"
"github.com/spf13/cobra"
@ -109,19 +110,21 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac
printTimestamp("From timestamp: %v (%v)", timestamp)
// Prepare progress bars
exportProgress := make(chan models.Progress)
importProgress := make(chan models.Progress)
progress := progressBar(exportProgress, importProgress)
progressChan := make(chan models.TransferProgress)
progress := setupProgressBars(progressChan)
wg := &sync.WaitGroup{}
// Export from source
exportChan := make(chan R, 1000)
go exp.Process(timestamp, exportChan, exportProgress)
go exp.Process(wg, timestamp, exportChan, progressChan)
// Import into target
resultChan := make(chan models.ImportResult)
go imp.Process(exportChan, resultChan, importProgress)
go imp.Process(wg, exportChan, resultChan, progressChan)
result := <-resultChan
progress.Wait()
wg.Wait()
progress.close()
// Update timestamp
err = c.updateTimestamp(&result, timestamp)

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -55,7 +55,7 @@ type ListensExport interface {
// Returns a list of all listens newer then oldestTimestamp.
// The returned list of listens is supposed to be ordered by the
// Listen.ListenedAt timestamp, with the oldest entry first.
ExportListens(oldestTimestamp time.Time, results chan ListensResult, progress chan Progress)
ExportListens(oldestTimestamp time.Time, results chan ListensResult, progress chan TransferProgress)
}
// Must be implemented by services supporting the import of listens.
@ -63,7 +63,7 @@ type ListensImport interface {
ImportBackend
// Imports the given list of listens.
ImportListens(export ListensResult, importResult ImportResult, progress chan Progress) (ImportResult, error)
ImportListens(export ListensResult, importResult ImportResult, progress chan TransferProgress) (ImportResult, error)
}
// Must be implemented by services supporting the export of loves.
@ -73,7 +73,7 @@ type LovesExport interface {
// Returns a list of all loves newer then oldestTimestamp.
// The returned list of listens is supposed to be ordered by the
// Love.Created timestamp, with the oldest entry first.
ExportLoves(oldestTimestamp time.Time, results chan LovesResult, progress chan Progress)
ExportLoves(oldestTimestamp time.Time, results chan LovesResult, progress chan TransferProgress)
}
// Must be implemented by services supporting the import of loves.
@ -81,5 +81,5 @@ type LovesImport interface {
ImportBackend
// Imports the given list of loves.
ImportLoves(export LovesResult, importResult ImportResult, progress chan Progress) (ImportResult, error)
ImportLoves(export LovesResult, importResult ImportResult, progress chan TransferProgress) (ImportResult, error)
}

View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
@ -209,11 +209,25 @@ func (i *ImportResult) Log(t LogEntryType, msg string) {
})
}
type TransferProgress struct {
Export *Progress
Import *Progress
}
func (p TransferProgress) FromImportResult(result ImportResult, completed bool) TransferProgress {
importProgress := Progress{
Completed: completed,
}.FromImportResult(result)
p.Import = &importProgress
return p
}
type Progress struct {
Total int64
Elapsed int64
Completed bool
Aborted bool
TotalItems int
Total int64
Elapsed int64
Completed bool
Aborted bool
}
func (p Progress) FromImportResult(result ImportResult) Progress {
@ -222,13 +236,11 @@ func (p Progress) FromImportResult(result ImportResult) Progress {
return p
}
func (p Progress) Complete() Progress {
func (p *Progress) Complete() {
p.Elapsed = p.Total
p.Completed = true
return p
}
func (p Progress) Abort() Progress {
func (p *Progress) Abort() {
p.Aborted = true
return p
}