marc/scotty
1
0
Fork 0
mirror of https://git.sr.ht/~phw/scotty synced 2025-04-30 21:57:06 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implemented progressbar for export/import

Browse source
This commit is contained in:
Philipp Wolfer 2023-11-16 00:45:00 +01:00
parent ab04eb1123
commit 6e330daf06
No known key found for this signature in database
GPG key ID: 8FDF744D4919943B
24 changed files with 590 additions and 239 deletions
Download patch file Download diff file Expand all files Collapse all files

144
backends/listenbrainz/listenbrainz.go
View file

@ -31,25 +31,34 @@ import (
)
type ListenBrainzApiBackend struct {
client Client
username string
client Client
username string
existingMbids map[string]bool
}
func (b ListenBrainzApiBackend) FromConfig(config *viper.Viper) models.Backend {
func (b *ListenBrainzApiBackend) FromConfig(config *viper.Viper) models.Backend {
b.client = NewClient(config.GetString("token"))
b.client.MaxResults = MaxItemsPerGet
b.username = config.GetString("username")
return b
}
func (b ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) {
maxTime := time.Now()
func (b *ListenBrainzApiBackend) Init() error { return nil }
func (b *ListenBrainzApiBackend) Finish() error { return nil }
func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
startTime := time.Now()
maxTime := startTime
minTime := time.Unix(0, 0)
totalDuration := startTime.Sub(oldestTimestamp)
defer close(results)
defer close(progress)
// FIXME: Optimize by fetching the listens in reverse listen time order
listens := make(models.ListensList, 0, 2*MaxItemsPerGet)
p := models.Progress{Total: int64(totalDuration.Seconds())}
out:
for {
@ -66,6 +75,7 @@ out:
// Set maxTime to the oldest returned listen
maxTime = time.Unix(result.Payload.Listens[count-1].ListenedAt, 0)
remainingTime := maxTime.Sub(oldestTimestamp)
for _, listen := range result.Payload.Listens {
if listen.ListenedAt > oldestTimestamp.Unix() {
@ -73,19 +83,26 @@ out:
} else {
// result contains listens older then oldestTimestamp,
// we can stop requesting more
p.Total = int64(startTime.Sub(time.Unix(listen.ListenedAt, 0)).Seconds())
break out
}
}
p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
progress <- p
}
sort.Sort(listens)
results <- models.ListensResult{Listens: listens}
progress <- p.Complete()
results <- models.ListensResult{Listens: listens, OldestTimestamp: oldestTimestamp}
}
func (b ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) {
func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) {
offset := 0
defer close(results)
defer close(progress)
loves := make(models.LovesList, 0, 2*MaxItemsPerGet)
p := models.Progress{}
out:
for {
@ -104,84 +121,77 @@ out:
love := feedback.ToLove()
if love.Created.Unix() > oldestTimestamp.Unix() {
loves = append(loves, love)
p.Elapsed += 1
progress <- p
} else {
break out
}
}
p.Total = int64(result.TotalCount)
p.Elapsed += int64(count)
offset += MaxItemsPerGet
}
sort.Sort(loves)
progress <- p.Complete()
results <- models.LovesResult{Loves: loves}
}
func (b ListenBrainzApiBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) {
importResult := models.ImportResult{
LastTimestamp: oldestTimestamp,
ImportErrors: make([]string, 0),
}
existingLovesChan := make(chan models.LovesResult)
go b.ExportLoves(time.Unix(0, 0), existingLovesChan)
existingLoves := <-existingLovesChan
if existingLoves.Error != nil {
results <- models.LovesResult{Error: existingLoves.Error}
close(results)
}
existingMbids := make(map[string]bool, len(existingLoves.Loves))
for _, love := range existingLoves.Loves {
existingMbids[string(love.RecordingMbid)] = true
}
for result := range results {
if result.Error != nil {
return importResult, result.Error
func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {
if len(b.existingMbids) == 0 {
existingLovesChan := make(chan models.LovesResult)
go b.ExportLoves(time.Unix(0, 0), existingLovesChan, progress)
existingLoves := <-existingLovesChan
if existingLoves.Error != nil {
return importResult, existingLoves.Error
}
importResult.TotalCount += len(result.Loves)
// TODO: Store MBIDs directly
b.existingMbids = make(map[string]bool, len(existingLoves.Loves))
for _, love := range existingLoves.Loves {
b.existingMbids[string(love.RecordingMbid)] = true
}
}
for _, love := range result.Loves {
if love.Created.Unix() <= oldestTimestamp.Unix() {
continue
}
for _, love := range export.Loves {
recordingMbid := string(love.RecordingMbid)
recordingMbid := string(love.RecordingMbid)
if recordingMbid == "" {
lookup, err := b.client.Lookup(love.TrackName, love.ArtistName())
if err == nil {
recordingMbid = lookup.RecordingMbid
}
}
if recordingMbid != "" {
ok := false
errMsg := ""
if existingMbids[recordingMbid] {
ok = true
} else {
resp, err := b.client.SendFeedback(Feedback{
RecordingMbid: recordingMbid,
Score: 1,
})
ok = err == nil && resp.Status == "ok"
if err != nil {
errMsg = err.Error()
}
}
if ok {
importResult.UpdateTimestamp(love.Created)
importResult.ImportCount += 1
} else {
msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v",
love.TrackName, love.ArtistName(), errMsg)
importResult.ImportErrors = append(importResult.ImportErrors, msg)
}
if recordingMbid == "" {
lookup, err := b.client.Lookup(love.TrackName, love.ArtistName())
if err == nil {
recordingMbid = lookup.RecordingMbid
}
}
if recordingMbid != "" {
ok := false
errMsg := ""
if b.existingMbids[recordingMbid] {
ok = true
} else {
resp, err := b.client.SendFeedback(Feedback{
RecordingMbid: recordingMbid,
Score: 1,
})
ok = err == nil && resp.Status == "ok"
if err != nil {
errMsg = err.Error()
}
}
if ok {
importResult.UpdateTimestamp(love.Created)
importResult.ImportCount += 1
} else {
msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v",
love.TrackName, love.ArtistName(), errMsg)
importResult.ImportErrors = append(importResult.ImportErrors, msg)
}
}
progress <- models.Progress{}.FromImportResult(importResult)
}
return importResult, nil