diff --git a/CHANGES.md b/CHANGES.md index c763e1d..615939f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Scotty Changelog +## 0.3.0 - unreleased +- listenbrainz: fetch listens in reverse listen time order + + ## 0.2.0 - 2023-11-28 - lastfm: support for scrobble and love export/import - jspf: consider loved track MBID diff --git a/internal/backends/listenbrainz/client_test.go b/internal/backends/listenbrainz/client_test.go index faabbe1..cc36f1d 100644 --- a/internal/backends/listenbrainz/client_test.go +++ b/internal/backends/listenbrainz/client_test.go @@ -53,6 +53,8 @@ func TestGetListens(t *testing.T) { assert := assert.New(t) assert.Equal(2, result.Payload.Count) + assert.Equal(int64(1699718723), result.Payload.LatestListenTimestamp) + assert.Equal(int64(1152911863), result.Payload.OldestListenTimestamp) require.Len(t, result.Payload.Listens, 2) assert.Equal("Shadowplay", result.Payload.Listens[0].TrackMetadata.TrackName) } diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index cafcc0f..0b29767 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -46,20 +46,19 @@ func (b *ListenBrainzApiBackend) FinishImport() error { return nil } func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { startTime := time.Now() - maxTime := startTime - minTime := time.Unix(0, 0) + minTime := oldestTimestamp + if minTime.Unix() < 1 { + minTime = time.Unix(1, 0) + } - totalDuration := startTime.Sub(oldestTimestamp) + totalDuration := startTime.Sub(minTime) defer close(results) - // FIXME: Optimize by fetching the listens in reverse listen time order - listens := make(models.ListensList, 0, 2*MaxItemsPerGet) p := models.Progress{Total: int64(totalDuration.Seconds())} -out: for { - result, err := b.client.GetListens(b.username, maxTime, minTime) + result, err := b.client.GetListens(b.username, time.Now(), minTime) if err != nil { progress <- p.Complete() results <- models.ListensResult{Error: err} @@ -68,31 +67,39 @@ out: count := len(result.Payload.Listens) if count == 0 { - break + if minTime.Unix() < result.Payload.OldestListenTimestamp { + minTime = time.Unix(result.Payload.OldestListenTimestamp, 0) + totalDuration = startTime.Sub(minTime) + p.Total = int64(totalDuration.Seconds()) + continue + } else { + break + } } - // Set maxTime to the oldest returned listen - maxTime = time.Unix(result.Payload.Listens[count-1].ListenedAt, 0) - remainingTime := maxTime.Sub(oldestTimestamp) + // Set minTime to the newest returned listen + minTime = time.Unix(result.Payload.Listens[0].ListenedAt, 0) + remainingTime := startTime.Sub(minTime) + + listens := make(models.ListensList, 0, count) for _, listen := range result.Payload.Listens { if listen.ListenedAt > oldestTimestamp.Unix() { listens = append(listens, listen.AsListen()) } else { - // result contains listens older then oldestTimestamp, - // we can stop requesting more - p.Total = int64(startTime.Sub(time.Unix(listen.ListenedAt, 0)).Seconds()) - break out + // result contains listens older then oldestTimestamp + break } } + sort.Sort(listens) p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) progress <- p + results <- models.ListensResult{Listens: listens, OldestTimestamp: minTime} } - sort.Sort(listens) + results <- models.ListensResult{OldestTimestamp: minTime} progress <- p.Complete() - results <- models.ListensResult{Listens: listens, OldestTimestamp: oldestTimestamp} } func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { diff --git a/internal/backends/listenbrainz/models.go b/internal/backends/listenbrainz/models.go index a7cd32b..c1552c7 100644 --- a/internal/backends/listenbrainz/models.go +++ b/internal/backends/listenbrainz/models.go @@ -36,6 +36,7 @@ type GetListenPayload struct { Count int `json:"count"` UserName string `json:"user_id"` LatestListenTimestamp int64 `json:"latest_listen_ts"` + OldestListenTimestamp int64 `json:"oldest_listen_ts"` Listens []Listen `json:"listens"` } diff --git a/internal/backends/listenbrainz/testdata/listens.json b/internal/backends/listenbrainz/testdata/listens.json index 5d799b8..41119ba 100644 --- a/internal/backends/listenbrainz/testdata/listens.json +++ b/internal/backends/listenbrainz/testdata/listens.json @@ -2,6 +2,7 @@ "payload": { "count": 2, "latest_listen_ts": 1699718723, + "oldest_listen_ts": 1152911863, "listens": [ { "inserted_at": 1699719320, diff --git a/internal/backends/spotify/spotify.go b/internal/backends/spotify/spotify.go index 44db61c..1c797c8 100644 --- a/internal/backends/spotify/spotify.go +++ b/internal/backends/spotify/spotify.go @@ -122,15 +122,14 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha break } - listens := make(models.ListensList, 0, len(result.Items)) + listens := make(models.ListensList, 0, count) for _, listen := range result.Items { l := listen.AsListen() if l.ListenedAt.Unix() > oldestTimestamp.Unix() { listens = append(listens, l) } else { - // result contains listens older then oldestTimestamp, - // we can stop requesting more + // result contains listens older then oldestTimestamp break } }