listenbrainz: fetch listens in reverse listen time order

This allows parallel import

CHANGES.md

@@ -1,5 +1,9 @@
 # Scotty Changelog
 
+## 0.3.0 - unreleased
+- listenbrainz: fetch listens in reverse listen time order
+
+
 ## 0.2.0 - 2023-11-28
 - lastfm: support for scrobble and love export/import
 - jspf: consider loved track MBID

internal/backends/listenbrainz/client_test.go

@@ -53,6 +53,8 @@ func TestGetListens(t *testing.T) {
 
 	assert := assert.New(t)
 	assert.Equal(2, result.Payload.Count)
+	assert.Equal(int64(1699718723), result.Payload.LatestListenTimestamp)
+	assert.Equal(int64(1152911863), result.Payload.OldestListenTimestamp)
 	require.Len(t, result.Payload.Listens, 2)
 	assert.Equal("Shadowplay", result.Payload.Listens[0].TrackMetadata.TrackName)
 }

internal/backends/listenbrainz/listenbrainz.go

@@ -46,20 +46,19 @@ func (b *ListenBrainzApiBackend) FinishImport() error { return nil }
 
 func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) {
 	startTime := time.Now()
-	maxTime := startTime
-	minTime := time.Unix(0, 0)
+	minTime := oldestTimestamp
+	if minTime.Unix() < 1 {
+		minTime = time.Unix(1, 0)
+	}
 
-	totalDuration := startTime.Sub(oldestTimestamp)
+	totalDuration := startTime.Sub(minTime)
 
 	defer close(results)
 
-	// FIXME: Optimize by fetching the listens in reverse listen time order
-	listens := make(models.ListensList, 0, 2*MaxItemsPerGet)
 	p := models.Progress{Total: int64(totalDuration.Seconds())}
 
-out:
 	for {
-		result, err := b.client.GetListens(b.username, maxTime, minTime)
+		result, err := b.client.GetListens(b.username, time.Now(), minTime)
 		if err != nil {
 			progress <- p.Complete()
 			results <- models.ListensResult{Error: err}
@@ -68,31 +67,39 @@ out:
 
 		count := len(result.Payload.Listens)
 		if count == 0 {
-			break
+			if minTime.Unix() < result.Payload.OldestListenTimestamp {
+				minTime = time.Unix(result.Payload.OldestListenTimestamp, 0)
+				totalDuration = startTime.Sub(minTime)
+				p.Total = int64(totalDuration.Seconds())
+				continue
+			} else {
+				break
+			}
 		}
 
-		// Set maxTime to the oldest returned listen
-		maxTime = time.Unix(result.Payload.Listens[count-1].ListenedAt, 0)
-		remainingTime := maxTime.Sub(oldestTimestamp)
+		// Set minTime to the newest returned listen
+		minTime = time.Unix(result.Payload.Listens[0].ListenedAt, 0)
+		remainingTime := startTime.Sub(minTime)
+
+		listens := make(models.ListensList, 0, count)
 
 		for _, listen := range result.Payload.Listens {
 			if listen.ListenedAt > oldestTimestamp.Unix() {
 				listens = append(listens, listen.AsListen())
 			} else {
-				// result contains listens older then oldestTimestamp,
-				// we can stop requesting more
-				p.Total = int64(startTime.Sub(time.Unix(listen.ListenedAt, 0)).Seconds())
-				break out
+				// result contains listens older then oldestTimestamp
+				break
 			}
 		}
 
+		sort.Sort(listens)
 		p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
 		progress <- p
+		results <- models.ListensResult{Listens: listens, OldestTimestamp: minTime}
 	}
 
-	sort.Sort(listens)
+	results <- models.ListensResult{OldestTimestamp: minTime}
 	progress <- p.Complete()
-	results <- models.ListensResult{Listens: listens, OldestTimestamp: oldestTimestamp}
 }
 
 func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) {

internal/backends/listenbrainz/models.go

@@ -36,6 +36,7 @@ type GetListenPayload struct {
 	Count                 int      `json:"count"`
 	UserName              string   `json:"user_id"`
 	LatestListenTimestamp int64    `json:"latest_listen_ts"`
+	OldestListenTimestamp int64    `json:"oldest_listen_ts"`
 	Listens               []Listen `json:"listens"`
 }
 

internal/backends/listenbrainz/testdata/listens.json

@@ -2,6 +2,7 @@
   "payload": {
     "count": 2,
     "latest_listen_ts": 1699718723,
+    "oldest_listen_ts": 1152911863,
     "listens": [
       {
         "inserted_at": 1699719320,

internal/backends/spotify/spotify.go

@@ -122,15 +122,14 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha
 			break
 		}
 
-		listens := make(models.ListensList, 0, len(result.Items))
+		listens := make(models.ListensList, 0, count)
 
 		for _, listen := range result.Items {
 			l := listen.AsListen()
 			if l.ListenedAt.Unix() > oldestTimestamp.Unix() {
 				listens = append(listens, l)
 			} else {
-				// result contains listens older then oldestTimestamp,
-				// we can stop requesting more
+				// result contains listens older then oldestTimestamp
 				break
 			}
 		}