marc/scotty
1
0
Fork 0
mirror of https://git.sr.ht/~phw/scotty synced 2025-05-11 18:47:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Initial implementation of unified export/import progress

Browse source 
Both export and import progress get updated over a unified channel.
Most importantly this allows updating the import total from latest
export results.
This commit is contained in:
Philipp Wolfer 2025-05-05 11:38:29 +02:00
parent 1f48abc284
commit b8e6ccffdb
No known key found for this signature in database
GPG key ID: 8FDF744D4919943B
18 changed files with 369 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

12
internal/backends/backends_test.go
View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -18,7 +18,6 @@ Scotty. If not, see <https://www.gnu.org/licenses/>.
package backends_test
import (
"reflect"
"testing"
"github.com/spf13/viper"
@ -33,6 +32,7 @@ import (
"go.uploadedlobster.com/scotty/internal/backends/maloja"
"go.uploadedlobster.com/scotty/internal/backends/scrobblerlog"
"go.uploadedlobster.com/scotty/internal/backends/spotify"
"go.uploadedlobster.com/scotty/internal/backends/spotifyhistory"
"go.uploadedlobster.com/scotty/internal/backends/subsonic"
"go.uploadedlobster.com/scotty/internal/config"
"go.uploadedlobster.com/scotty/internal/i18n"
@ -93,9 +93,9 @@ func TestImplementsInterfaces(t *testing.T) {
expectInterface[models.LovesExport](t, &funkwhale.FunkwhaleApiBackend{})
// expectInterface[models.LovesImport](t, &funkwhale.FunkwhaleApiBackend{})
// expectInterface[models.ListensExport](t, &jspf.JSPFBackend{})
expectInterface[models.ListensExport](t, &jspf.JSPFBackend{})
expectInterface[models.ListensImport](t, &jspf.JSPFBackend{})
// expectInterface[models.LovesExport](t, &jspf.JSPFBackend{})
expectInterface[models.LovesExport](t, &jspf.JSPFBackend{})
expectInterface[models.LovesImport](t, &jspf.JSPFBackend{})
// expectInterface[models.ListensExport](t, &lastfm.LastfmApiBackend{})
@ -115,6 +115,8 @@ func TestImplementsInterfaces(t *testing.T) {
expectInterface[models.LovesExport](t, &spotify.SpotifyApiBackend{})
// expectInterface[models.LovesImport](t, &spotify.SpotifyApiBackend{})
expectInterface[models.ListensExport](t, &spotifyhistory.SpotifyHistoryBackend{})
expectInterface[models.ListensExport](t, &scrobblerlog.ScrobblerLogBackend{})
expectInterface[models.ListensImport](t, &scrobblerlog.ScrobblerLogBackend{})
@ -125,6 +127,6 @@ func TestImplementsInterfaces(t *testing.T) {
func expectInterface[T interface{}](t *testing.T, backend models.Backend) {
ok, name := backends.ImplementsInterface[T](&backend)
if !ok {
t.Errorf("%v expected to implement %v", reflect.TypeOf(backend).Name(), name)
t.Errorf("%v expected to implement %v", backend.Name(), name)
}
}

41
internal/backends/deezer/deezer.go
View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
Scotty is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
@ -77,7 +77,7 @@ func (b *DeezerApiBackend) OAuth2Setup(token oauth2.TokenSource) error {
return nil
}
func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
// Choose a high offset, we attempt to search the loves backwards starting
// at the oldest one.
offset := math.MaxInt32
@ -88,13 +88,18 @@ func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan
totalDuration := startTime.Sub(oldestTimestamp)
p := models.Progress{Total: int64(totalDuration.Seconds())}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(totalDuration.Seconds()),
},
}
out:
for {
result, err := b.client.UserHistory(offset, perPage)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -102,7 +107,6 @@ out:
// The offset was higher then the actual number of tracks. Adjust the offset
// and continue.
if offset >= result.Total {
p.Total = int64(result.Total)
offset = max(result.Total-perPage, 0)
continue
}
@ -128,7 +132,8 @@ out:
}
remainingTime := startTime.Sub(minTime)
p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
p.Export.TotalItems += len(listens)
p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
progress <- p
results <- models.ListensResult{Items: listens, OldestTimestamp: minTime}
@ -144,23 +149,29 @@ out:
}
results <- models.ListensResult{OldestTimestamp: minTime}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (b *DeezerApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *DeezerApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
// Choose a high offset, we attempt to search the loves backwards starting
// at the oldest one.
offset := math.MaxInt32
perPage := MaxItemsPerGet
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
var totalCount int
out:
for {
result, err := b.client.UserTracks(offset, perPage)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
@ -168,8 +179,8 @@ out:
// The offset was higher then the actual number of tracks. Adjust the offset
// and continue.
if offset >= result.Total {
p.Total = int64(result.Total)
totalCount = result.Total
p.Export.Total = int64(totalCount)
offset = max(result.Total-perPage, 0)
continue
}
@ -186,13 +197,14 @@ out:
loves = append(loves, love)
} else {
totalCount -= 1
break
}
}
sort.Sort(loves)
results <- models.LovesResult{Items: loves, Total: totalCount}
p.Elapsed += int64(count)
p.Export.TotalItems = totalCount
p.Export.Total = int64(totalCount)
p.Export.Elapsed += int64(count)
progress <- p
if offset <= 0 {
@ -206,7 +218,8 @@ out:
}
}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (t Listen) AsListen() models.Listen {

10
internal/backends/dump/dump.go
View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -36,27 +36,27 @@ func (b *DumpBackend) InitConfig(config *config.ServiceConfig) error {
func (b *DumpBackend) StartImport() error { return nil }
func (b *DumpBackend) FinishImport() error { return nil }
func (b *DumpBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *DumpBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
for _, listen := range export.Items {
importResult.UpdateTimestamp(listen.ListenedAt)
importResult.ImportCount += 1
msg := fmt.Sprintf("🎶 %v: \"%v\" by %v (%v)",
listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMBID)
importResult.Log(models.Info, msg)
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
}
return importResult, nil
}
func (b *DumpBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *DumpBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
for _, love := range export.Items {
importResult.UpdateTimestamp(love.Created)
importResult.ImportCount += 1
msg := fmt.Sprintf("❤️ %v: \"%v\" by %v (%v)",
love.Created, love.TrackName, love.ArtistName(), love.RecordingMBID)
importResult.Log(models.Info, msg)
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
}
return importResult, nil

8
internal/backends/export.go
View file

@ -23,7 +23,7 @@ import (
type ExportProcessor[T models.ListensResult | models.LovesResult] interface {
ExportBackend() models.Backend
Process(oldestTimestamp time.Time, results chan T, progress chan models.Progress)
Process(oldestTimestamp time.Time, results chan T, progress chan models.TransferProgress)
}
type ListensExportProcessor struct {
@ -34,9 +34,8 @@ func (p ListensExportProcessor) ExportBackend() models.Backend {
return p.Backend
}
func (p ListensExportProcessor) Process(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (p ListensExportProcessor) Process(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
defer close(results)
defer close(progress)
p.Backend.ExportListens(oldestTimestamp, results, progress)
}
@ -48,8 +47,7 @@ func (p LovesExportProcessor) ExportBackend() models.Backend {
return p.Backend
}
func (p LovesExportProcessor) Process(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (p LovesExportProcessor) Process(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
defer close(results)
defer close(progress)
p.Backend.ExportLoves(oldestTimestamp, results, progress)
}

46
internal/backends/funkwhale/funkwhale.go
View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -60,19 +60,26 @@ func (b *FunkwhaleApiBackend) InitConfig(config *config.ServiceConfig) error {
return nil
}
func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
page := 1
perPage := MaxItemsPerGet
// We need to gather the full list of listens in order to sort them
listens := make(models.ListensList, 0, 2*perPage)
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
out:
for {
result, err := b.client.GetHistoryListenings(b.username, page, perPage)
if err != nil {
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
count := len(result.Results)
@ -83,7 +90,7 @@ out:
for _, fwListen := range result.Results {
listen := fwListen.AsListen()
if listen.ListenedAt.After(oldestTimestamp) {
p.Elapsed += 1
p.Export.Elapsed += 1
listens = append(listens, listen)
} else {
break out
@ -92,34 +99,42 @@ out:
if result.Next == "" {
// No further results
p.Total = p.Elapsed
p.Total -= int64(perPage - count)
p.Export.Total = p.Export.Elapsed
p.Export.Total -= int64(perPage - count)
break out
}
p.Total += int64(perPage)
p.Export.TotalItems = len(listens)
p.Export.Total += int64(perPage)
progress <- p
page += 1
}
sort.Sort(listens)
progress <- p.Complete()
p.Export.TotalItems = len(listens)
p.Export.Complete()
progress <- p
results <- models.ListensResult{Items: listens}
}
func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
page := 1
perPage := MaxItemsPerGet
// We need to gather the full list of listens in order to sort them
loves := make(models.LovesList, 0, 2*perPage)
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
out:
for {
result, err := b.client.GetFavoriteTracks(page, perPage)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
@ -132,7 +147,7 @@ out:
for _, favorite := range result.Results {
love := favorite.AsLove()
if love.Created.After(oldestTimestamp) {
p.Elapsed += 1
p.Export.Elapsed += 1
loves = append(loves, love)
} else {
break out
@ -144,13 +159,16 @@ out:
break out
}
p.Total += int64(perPage)
p.Export.TotalItems = len(loves)
p.Export.Total += int64(perPage)
progress <- p
page += 1
}
sort.Sort(loves)
progress <- p.Complete()
p.Export.TotalItems = len(loves)
p.Export.Complete()
progress <- p
results <- models.LovesResult{Items: loves}
}

27
internal/backends/import.go
View file

@ -23,8 +23,8 @@ import (
type ImportProcessor[T models.ListensResult | models.LovesResult] interface {
ImportBackend() models.ImportBackend
Process(results chan T, out chan models.ImportResult, progress chan models.Progress)
Import(export T, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error)
Process(results chan T, out chan models.ImportResult, progress chan models.TransferProgress)
Import(export T, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error)
}
type ListensImportProcessor struct {
@ -35,11 +35,11 @@ func (p ListensImportProcessor) ImportBackend() models.ImportBackend {
return p.Backend
}
func (p ListensImportProcessor) Process(results chan models.ListensResult, out chan models.ImportResult, progress chan models.Progress) {
func (p ListensImportProcessor) Process(results chan models.ListensResult, out chan models.ImportResult, progress chan models.TransferProgress) {
process(p, results, out, progress)
}
func (p ListensImportProcessor) Import(export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (p ListensImportProcessor) Import(export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
if export.Error != nil {
return result, export.Error
}
@ -64,11 +64,11 @@ func (p LovesImportProcessor) ImportBackend() models.ImportBackend {
return p.Backend
}
func (p LovesImportProcessor) Process(results chan models.LovesResult, out chan models.ImportResult, progress chan models.Progress) {
func (p LovesImportProcessor) Process(results chan models.LovesResult, out chan models.ImportResult, progress chan models.TransferProgress) {
process(p, results, out, progress)
}
func (p LovesImportProcessor) Import(export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (p LovesImportProcessor) Import(export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
if export.Error != nil {
return result, export.Error
}
@ -85,11 +85,10 @@ func (p LovesImportProcessor) Import(export models.LovesResult, result models.Im
return importResult, nil
}
func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](processor P, results chan R, out chan models.ImportResult, progress chan models.Progress) {
func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](processor P, results chan R, out chan models.ImportResult, progress chan models.TransferProgress) {
defer close(out)
defer close(progress)
result := models.ImportResult{}
p := models.Progress{}
p := models.TransferProgress{}
if err := processor.ImportBackend().StartImport(); err != nil {
out <- handleError(result, err, progress)
@ -104,7 +103,7 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](
out <- handleError(result, err, progress)
return
}
progress <- p.FromImportResult(result)
progress <- p.FromImportResult(result, false)
}
if err := processor.ImportBackend().FinishImport(); err != nil {
@ -112,12 +111,14 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](
return
}
progress <- p.FromImportResult(result).Complete()
progress <- p.FromImportResult(result, true)
out <- result
}
func handleError(result models.ImportResult, err error, progress chan models.Progress) models.ImportResult {
func handleError(result models.ImportResult, err error, progress chan models.TransferProgress) models.ImportResult {
result.Error = err
progress <- models.Progress{}.FromImportResult(result).Abort()
p := models.TransferProgress{}.FromImportResult(result, false)
p.Import.Abort()
progress <- p
return result
}

34
internal/backends/jspf/jspf.go
View file

@ -93,10 +93,15 @@ func (b *JSPFBackend) FinishImport() error {
return b.writeJSPF()
}
func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
err := b.readJSPF()
p := models.TransferProgress{
Export: &models.Progress{},
}
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -109,11 +114,13 @@ func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan mode
}
}
sort.Sort(listens)
progress <- models.Progress{Total: int64(len(listens))}.Complete()
p.Export.Total = int64(len(listens))
p.Export.Complete()
progress <- p
results <- models.ListensResult{Items: listens}
}
func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
for _, listen := range export.Items {
track := listenAsTrack(listen)
b.playlist.Tracks = append(b.playlist.Tracks, track)
@ -121,14 +128,19 @@ func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult mo
importResult.UpdateTimestamp(listen.ListenedAt)
}
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
return importResult, nil
}
func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
err := b.readJSPF()
p := models.TransferProgress{
Export: &models.Progress{},
}
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
@ -141,11 +153,13 @@ func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models
}
}
sort.Sort(loves)
progress <- models.Progress{Total: int64(len(loves))}.Complete()
p.Export.Total = int64(len(loves))
p.Export.Complete()
progress <- p
results <- models.LovesResult{Items: loves}
}
func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
for _, love := range export.Items {
track := loveAsTrack(love)
b.playlist.Tracks = append(b.playlist.Tracks, track)
@ -153,7 +167,7 @@ func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models
importResult.UpdateTimestamp(love.Created)
}
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
return importResult, nil
}

56
internal/backends/lastfm/lastfm.go
View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
Scotty is free software: you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
@ -88,13 +88,17 @@ func (b *LastfmApiBackend) OAuth2Setup(token oauth2.TokenSource) error {
return nil
}
func (b *LastfmApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *LastfmApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
page := MaxPage
minTime := oldestTimestamp
perPage := MaxItemsPerGet
// We need to gather the full list of listens in order to sort them
p := models.Progress{Total: int64(page)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(page),
},
}
out:
for page > 0 {
@ -108,7 +112,8 @@ out:
result, err := b.client.User.GetRecentTracks(args)
if err != nil {
results <- models.ListensResult{Error: err}
progress <- p.Abort()
p.Export.Abort()
progress <- p
return
}
@ -127,11 +132,12 @@ out:
timestamp, err := strconv.ParseInt(scrobble.Date.Uts, 10, 64)
if err != nil {
results <- models.ListensResult{Error: err}
progress <- p.Abort()
p.Export.Abort()
progress <- p
break out
}
if timestamp > oldestTimestamp.Unix() {
p.Elapsed += 1
p.Export.Elapsed += 1
listen := models.Listen{
ListenedAt: time.Unix(timestamp, 0),
UserName: b.username,
@ -165,16 +171,18 @@ out:
Total: result.Total,
OldestTimestamp: minTime,
}
p.Total = int64(result.TotalPages)
p.Elapsed = int64(result.TotalPages - page)
p.Export.Total = int64(result.TotalPages)
p.Export.Elapsed = int64(result.TotalPages - page)
p.Export.TotalItems += len(listens)
progress <- p
}
results <- models.ListensResult{OldestTimestamp: minTime}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
total := len(export.Items)
for i := 0; i < total; i += MaxListensPerSubmission {
listens := export.Items[i:min(i+MaxListensPerSubmission, total)]
@ -244,20 +252,24 @@ func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResu
importResult.UpdateTimestamp(listens[count-1].ListenedAt)
importResult.ImportCount += accepted
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
}
return importResult, nil
}
func (b *LastfmApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *LastfmApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
// Choose a high offset, we attempt to search the loves backwards starting
// at the oldest one.
page := 1
perPage := MaxItemsPerGet
loves := make(models.LovesList, 0, 2*MaxItemsPerGet)
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
var totalCount int
out:
@ -268,12 +280,12 @@ out:
"page": page,
})
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
p.Total = int64(result.Total)
count := len(result.Tracks)
if count == 0 {
break out
@ -282,7 +294,8 @@ out:
for _, track := range result.Tracks {
timestamp, err := strconv.ParseInt(track.Date.Uts, 10, 64)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
@ -308,18 +321,21 @@ out:
}
}
p.Elapsed += int64(count)
p.Export.Total += int64(perPage)
p.Export.TotalItems = totalCount
p.Export.Elapsed += int64(count)
progress <- p
page += 1
}
sort.Sort(loves)
p.Export.Complete()
progress <- p
results <- models.LovesResult{Items: loves, Total: totalCount}
progress <- p.Complete()
}
func (b *LastfmApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *LastfmApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
for _, love := range export.Items {
err := b.client.Track.Love(lastfm.P{
"track": love.TrackName,
@ -335,7 +351,7 @@ func (b *LastfmApiBackend) ImportLoves(export models.LovesResult, importResult m
importResult.Log(models.Error, msg)
}
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
}
return importResult, nil

53
internal/backends/listenbrainz/listenbrainz.go
View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -72,21 +72,25 @@ func (b *ListenBrainzApiBackend) InitConfig(config *config.ServiceConfig) error
func (b *ListenBrainzApiBackend) StartImport() error { return nil }
func (b *ListenBrainzApiBackend) FinishImport() error { return nil }
func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
startTime := time.Now()
minTime := oldestTimestamp
if minTime.Unix() < 1 {
minTime = time.Unix(1, 0)
}
totalDuration := startTime.Sub(minTime)
p := models.Progress{Total: int64(totalDuration.Seconds())}
totalDuration := startTime.Sub(oldestTimestamp)
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(totalDuration.Seconds()),
},
}
for {
result, err := b.client.GetListens(b.username, time.Now(), minTime)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -96,7 +100,7 @@ func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, result
if minTime.Unix() < result.Payload.OldestListenTimestamp {
minTime = time.Unix(result.Payload.OldestListenTimestamp, 0)
totalDuration = startTime.Sub(minTime)
p.Total = int64(totalDuration.Seconds())
p.Export.Total = int64(totalDuration.Seconds())
continue
} else {
break
@ -119,18 +123,20 @@ func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, result
}
sort.Sort(listens)
p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
p.Export.TotalItems += len(listens)
p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
progress <- p
results <- models.ListensResult{Items: listens, OldestTimestamp: minTime}
}
results <- models.ListensResult{OldestTimestamp: minTime}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
total := len(export.Items)
p := models.Progress{}.FromImportResult(importResult)
p := models.TransferProgress{}.FromImportResult(importResult, false)
for i := 0; i < total; i += MaxListensPerRequest {
listens := export.Items[i:min(i+MaxListensPerRequest, total)]
count := len(listens)
@ -146,7 +152,7 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo
for _, l := range listens {
if b.checkDuplicates {
isDupe, err := b.checkDuplicateListen(l)
p.Elapsed += 1
p.Import.Elapsed += 1
progress <- p
if err != nil {
return importResult, err
@ -186,31 +192,36 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo
importResult.UpdateTimestamp(listens[count-1].ListenedAt)
}
importResult.ImportCount += count
progress <- p.FromImportResult(importResult)
progress <- p.FromImportResult(importResult, false)
}
return importResult, nil
}
func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
exportChan := make(chan models.LovesResult)
p := models.Progress{}
p := models.TransferProgress{
Export: &models.Progress{},
}
go b.exportLoves(oldestTimestamp, exportChan)
for existingLoves := range exportChan {
if existingLoves.Error != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: existingLoves.Error}
return
}
p.Total = int64(existingLoves.Total)
p.Elapsed += int64(existingLoves.Items.Len())
p.Export.TotalItems = existingLoves.Total
p.Export.Total = int64(existingLoves.Total)
p.Export.Elapsed += int64(len(existingLoves.Items))
progress <- p
results <- existingLoves
}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (b *ListenBrainzApiBackend) exportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
@ -260,7 +271,7 @@ out:
}
}
func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
if len(b.existingMBIDs) == 0 {
existingLovesChan := make(chan models.LovesResult)
go b.exportLoves(time.Unix(0, 0), existingLovesChan)
@ -330,7 +341,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importRe
importResult.Log(models.Error, msg)
}
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
}
return importResult, nil

27
internal/backends/maloja/maloja.go
View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -63,19 +63,24 @@ func (b *MalojaApiBackend) InitConfig(config *config.ServiceConfig) error {
func (b *MalojaApiBackend) StartImport() error { return nil }
func (b *MalojaApiBackend) FinishImport() error { return nil }
func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
page := 0
perPage := MaxItemsPerGet
// We need to gather the full list of listens in order to sort them
listens := make(models.ListensList, 0, 2*perPage)
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
out:
for {
result, err := b.client.GetScrobbles(page, perPage)
if err != nil {
progress <- p.Complete()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -87,25 +92,27 @@ out:
for _, scrobble := range result.List {
if scrobble.ListenedAt > oldestTimestamp.Unix() {
p.Elapsed += 1
p.Export.Elapsed += 1
listens = append(listens, scrobble.AsListen())
} else {
break out
}
}
p.Total += int64(perPage)
p.Export.TotalItems = len(listens)
p.Export.Total += int64(perPage)
progress <- p
page += 1
}
sort.Sort(listens)
progress <- p.Complete()
p.Export.Complete()
progress <- p
results <- models.ListensResult{Items: listens}
}
func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
p := models.Progress{}.FromImportResult(importResult)
func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
p := models.TransferProgress{}.FromImportResult(importResult, false)
for _, listen := range export.Items {
scrobble := NewScrobble{
Title: listen.TrackName,
@ -126,7 +133,7 @@ func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResu
importResult.UpdateTimestamp(listen.ListenedAt)
importResult.ImportCount += 1
progress <- p.FromImportResult(importResult)
progress <- p.FromImportResult(importResult, false)
}
return importResult, nil

19
internal/backends/scrobblerlog/scrobblerlog.go
View file

@ -131,10 +131,14 @@ func (b *ScrobblerLogBackend) FinishImport() error {
return b.file.Close()
}
func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
file, err := os.Open(b.filePath)
p := models.TransferProgress{
Export: &models.Progress{},
}
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -143,7 +147,8 @@ func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results c
err = b.log.Parse(file, b.ignoreSkipped)
if err != nil {
progress <- models.Progress{}.Complete()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -157,11 +162,13 @@ func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results c
}
}
sort.Sort(listens)
progress <- models.Progress{Total: int64(len(listens))}.Complete()
p.Export.Total = int64(len(listens))
p.Export.Complete()
progress <- p
results <- models.ListensResult{Items: listens}
}
func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
records := make([]scrobblerlog.Record, len(export.Items))
for i, listen := range export.Items {
records[i] = listenToRecord(listen)
@ -173,7 +180,7 @@ func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importR
importResult.UpdateTimestamp(lastTimestamp)
importResult.ImportCount += len(export.Items)
progress <- models.Progress{}.FromImportResult(importResult)
progress <- models.TransferProgress{}.FromImportResult(importResult, false)
return importResult, nil
}

41
internal/backends/spotify/spotify.go
View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -95,18 +95,22 @@ func (b *SpotifyApiBackend) OAuth2Setup(token oauth2.TokenSource) error {
return nil
}
func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
startTime := time.Now()
minTime := oldestTimestamp
totalDuration := startTime.Sub(oldestTimestamp)
p := models.Progress{Total: int64(totalDuration.Seconds())}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(totalDuration.Seconds()),
},
}
for {
result, err := b.client.RecentlyPlayedAfter(minTime, MaxItemsPerGet)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -118,7 +122,8 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha
// Set minTime to the newest returned listen
after, err := strconv.ParseInt(result.Cursors.After, 10, 64)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
} else if after <= minTime.Unix() {
@ -146,22 +151,28 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha
}
sort.Sort(listens)
p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
p.Export.TotalItems += len(listens)
p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
progress <- p
results <- models.ListensResult{Items: listens, OldestTimestamp: minTime}
}
results <- models.ListensResult{OldestTimestamp: minTime}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (b *SpotifyApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *SpotifyApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
// Choose a high offset, we attempt to search the loves backwards starting
// at the oldest one.
offset := math.MaxInt32
perPage := MaxItemsPerGet
p := models.Progress{Total: int64(perPage)}
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(perPage),
},
}
totalCount := 0
exportCount := 0
@ -169,7 +180,8 @@ out:
for {
result, err := b.client.UserTracks(offset, perPage)
if err != nil {
progress <- p.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
@ -177,7 +189,7 @@ out:
// The offset was higher then the actual number of tracks. Adjust the offset
// and continue.
if offset >= result.Total {
p.Total = int64(result.Total)
p.Export.Total = int64(result.Total)
totalCount = result.Total
offset = max(result.Total-perPage, 0)
continue
@ -201,7 +213,7 @@ out:
exportCount += len(loves)
sort.Sort(loves)
results <- models.LovesResult{Items: loves, Total: totalCount}
p.Elapsed += int64(count)
p.Export.Elapsed += int64(count)
progress <- p
if offset <= 0 {
@ -216,7 +228,8 @@ out:
}
results <- models.LovesResult{Total: exportCount}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func (l Listen) AsListen() models.Listen {

22
internal/backends/spotifyhistory/spotifyhistory.go
View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -72,21 +72,27 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error {
return nil
}
func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) {
files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob))
p := models.TransferProgress{
Export: &models.Progress{},
}
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
slices.Sort(files)
fileCount := int64(len(files))
p := models.Progress{Total: fileCount}
p.Export.Total = fileCount
for i, filePath := range files {
history, err := readHistoryFile(filePath)
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.ListensResult{Error: err}
return
}
@ -97,11 +103,13 @@ func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results
})
sort.Sort(listens)
results <- models.ListensResult{Items: listens}
p.Elapsed = int64(i)
p.Export.Elapsed = int64(i)
p.Export.TotalItems += len(listens)
progress <- p
}
progress <- p.Complete()
p.Export.Complete()
progress <- p
}
func readHistoryFile(filePath string) (StreamingHistory, error) {

19
internal/backends/subsonic/subsonic.go
View file

@ -1,5 +1,5 @@
/*
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
Copyright © 2023-2025 Philipp Wolfer <phw@uploadedlobster.com>
This file is part of Scotty.
@ -63,25 +63,30 @@ func (b *SubsonicApiBackend) InitConfig(config *config.ServiceConfig) error {
return nil
}
func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) {
err := b.client.Authenticate(b.password)
p := models.TransferProgress{
Export: &models.Progress{},
}
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
starred, err := b.client.GetStarred2(map[string]string{})
if err != nil {
progress <- models.Progress{}.Abort()
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
loves := b.filterSongs(starred.Song, oldestTimestamp)
progress <- models.Progress{
Total: int64(loves.Len()),
}.Complete()
p.Export.Total = int64(len(loves))
p.Export.Complete()
progress <- p
results <- models.LovesResult{Items: loves}
}