marc/scotty
1
0
Fork 0
mirror of https://git.sr.ht/~phw/scotty synced 2025-05-31 10:58:35 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implemented lbarchive loves export

Browse source
This commit is contained in:
Philipp Wolfer 2025-05-24 11:59:35 +02:00
parent d250952678
commit dddd2e4eec
No known key found for this signature in database
GPG key ID: 8FDF744D4919943B
5 changed files with 210 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -15,6 +15,7 @@ require (
github.com/manifoldco/promptui v0.9.0
github.com/pelletier/go-toml/v2 v2.2.4
github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0
github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740
github.com/spf13/cast v1.8.0
github.com/spf13/cobra v1.9.1
github.com/spf13/viper v1.20.1
@ -53,7 +54,6 @@ require (
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/sagikazarmark/locafero v0.9.0 // indirect
github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.14.0 // indirect
github.com/spf13/pflag v1.0.6 // indirect

2
internal/backends/backends_test.go
View file

@ -106,7 +106,7 @@ func TestImplementsInterfaces(t *testing.T) {
expectInterface[models.ListensExport](t, &lbarchive.ListenBrainzArchiveBackend{})
// expectInterface[models.ListensImport](t, &lbarchive.ListenBrainzArchiveBackend{})
// expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{})
expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{})
// expectInterface[models.LovesImport](t, &lbarchive.ListenBrainzArchiveBackend{})
expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{})

95
internal/backends/lbarchive/lbarchive.go
View file

@ -25,17 +25,23 @@ import (
"context"
"time"
"go.uploadedlobster.com/musicbrainzws2"
lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz"
"go.uploadedlobster.com/scotty/internal/config"
"go.uploadedlobster.com/scotty/internal/i18n"
"go.uploadedlobster.com/scotty/internal/listenbrainz"
"go.uploadedlobster.com/scotty/internal/models"
"go.uploadedlobster.com/scotty/internal/version"
)
const batchSize = 2000
const (
listensBatchSize = 2000
lovesBatchSize = 10
)
type ListenBrainzArchiveBackend struct {
filePath string
mbClient musicbrainzws2.Client
}
func (b *ListenBrainzArchiveBackend) Name() string { return "listenbrainz-archive" }
@ -50,6 +56,11 @@ func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption {
func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error {
b.filePath = config.GetString("file-path")
b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{
Name: version.AppName,
Version: version.AppVersion,
URL: version.AppURL,
})
return nil
}
@ -86,7 +97,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens(
return
}
listens := make(models.ListensList, 0, batchSize)
listens := make(models.ListensList, 0, listensBatchSize)
for rawListen, err := range archive.IterListens(oldestTimestamp) {
if err != nil {
p.Export.Abort()
@ -108,7 +119,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens(
// Allow the importer to start processing the listens by
// sending them in batches.
if len(listens) >= batchSize {
if len(listens) >= listensBatchSize {
results <- models.ListensResult{Items: listens}
progress <- p
listens = listens[:0]
@ -119,3 +130,81 @@ func (b *ListenBrainzArchiveBackend) ExportListens(
p.Export.Complete()
progress <- p
}
func (b *ListenBrainzArchiveBackend) ExportLoves(
ctx context.Context, oldestTimestamp time.Time,
results chan models.LovesResult, progress chan models.TransferProgress) {
startTime := time.Now()
minTime := oldestTimestamp
if minTime.Unix() < 1 {
minTime = time.Unix(1, 0)
}
totalDuration := startTime.Sub(oldestTimestamp)
p := models.TransferProgress{
Export: &models.Progress{
Total: int64(totalDuration.Seconds()),
},
}
archive, err := listenbrainz.OpenExportArchive(b.filePath)
if err != nil {
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
defer archive.Close()
userInfo, err := archive.UserInfo()
if err != nil {
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
loves := make(models.LovesList, 0, lovesBatchSize)
for feedback, err := range archive.IterFeedback(oldestTimestamp) {
if err != nil {
p.Export.Abort()
progress <- p
results <- models.LovesResult{Error: err}
return
}
// The export file does not include track metadata. Try fetching details
// from MusicBrainz.
if feedback.TrackMetadata == nil {
track, err := lbapi.LookupRecording(ctx, &b.mbClient, feedback.RecordingMBID)
if err == nil {
feedback.TrackMetadata = track
}
}
love := lbapi.AsLove(feedback)
if love.UserName == "" {
love.UserName = userInfo.Name
}
// TODO: The dump does not contain TrackMetadata for feedback.
// We need to look it up in the archive.
loves = append(loves, love)
// Update the progress
p.Export.TotalItems += 1
remainingTime := startTime.Sub(love.Created)
p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds())
// Allow the importer to start processing the listens by
// sending them in batches.
if len(loves) >= lovesBatchSize {
results <- models.LovesResult{Items: loves}
progress <- p
loves = loves[:0]
}
}
results <- models.LovesResult{Items: loves}
p.Export.Complete()
progress <- p
}

115
internal/backends/listenbrainz/helper.go Normal file
View file

@ -0,0 +1,115 @@
/*
Copyright © 2025 Philipp Wolfer <phw@uploadedlobster.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package listenbrainz
import (
"context"
"time"
"go.uploadedlobster.com/mbtypes"
"go.uploadedlobster.com/musicbrainzws2"
"go.uploadedlobster.com/scotty/internal/listenbrainz"
"go.uploadedlobster.com/scotty/internal/models"
)
func LookupRecording(
ctx context.Context,
mb *musicbrainzws2.Client,
mbid mbtypes.MBID,
) (*listenbrainz.Track, error) {
filter := musicbrainzws2.IncludesFilter{
Includes: []string{"artist-credits"},
}
recording, err := mb.LookupRecording(ctx, mbid, filter)
if err != nil {
return nil, err
}
artistMBIDs := make([]mbtypes.MBID, 0, len(recording.ArtistCredit))
for _, artist := range recording.ArtistCredit {
artistMBIDs = append(artistMBIDs, artist.Artist.ID)
}
track := listenbrainz.Track{
TrackName: recording.Title,
ArtistName: recording.ArtistCredit.String(),
MBIDMapping: &listenbrainz.MBIDMapping{
// In case of redirects this MBID differs from the looked up MBID
RecordingMBID: recording.ID,
ArtistMBIDs: artistMBIDs,
},
}
return &track, nil
}
func AsListen(lbListen listenbrainz.Listen) models.Listen {
listen := models.Listen{
ListenedAt: time.Unix(lbListen.ListenedAt, 0),
UserName: lbListen.UserName,
Track: AsTrack(lbListen.TrackMetadata),
}
return listen
}
func AsLove(f listenbrainz.Feedback) models.Love {
recordingMBID := f.RecordingMBID
track := f.TrackMetadata
if track == nil {
track = &listenbrainz.Track{}
}
love := models.Love{
UserName: f.UserName,
RecordingMBID: recordingMBID,
Created: time.Unix(f.Created, 0),
Track: AsTrack(*track),
}
if love.Track.RecordingMBID == "" {
love.Track.RecordingMBID = love.RecordingMBID
}
return love
}
func AsTrack(t listenbrainz.Track) models.Track {
track := models.Track{
TrackName: t.TrackName,
ReleaseName: t.ReleaseName,
ArtistNames: []string{t.ArtistName},
Duration: t.Duration(),
TrackNumber: t.TrackNumber(),
DiscNumber: t.DiscNumber(),
RecordingMBID: t.RecordingMBID(),
ReleaseMBID: t.ReleaseMBID(),
ReleaseGroupMBID: t.ReleaseGroupMBID(),
ISRC: t.ISRC(),
AdditionalInfo: t.AdditionalInfo,
}
if t.MBIDMapping != nil && len(track.ArtistMBIDs) == 0 {
for _, artistMBID := range t.MBIDMapping.ArtistMBIDs {
track.ArtistMBIDs = append(track.ArtistMBIDs, artistMBID)
}
}
return track
}

81
internal/backends/listenbrainz/listenbrainz.go
View file

@ -249,7 +249,7 @@ out:
// longer available and might have been merged. Try fetching details
// from MusicBrainz.
if feedback.TrackMetadata == nil {
track, err := b.lookupRecording(ctx, feedback.RecordingMBID)
track, err := LookupRecording(ctx, &b.mbClient, feedback.RecordingMBID)
if err == nil {
feedback.TrackMetadata = track
}
@ -375,82 +375,3 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste
return false, nil
}
func (b *ListenBrainzApiBackend) lookupRecording(
ctx context.Context, mbid mbtypes.MBID) (*listenbrainz.Track, error) {
filter := musicbrainzws2.IncludesFilter{
Includes: []string{"artist-credits"},
}
recording, err := b.mbClient.LookupRecording(ctx, mbid, filter)
if err != nil {
return nil, err
}
artistMBIDs := make([]mbtypes.MBID, 0, len(recording.ArtistCredit))
for _, artist := range recording.ArtistCredit {
artistMBIDs = append(artistMBIDs, artist.Artist.ID)
}
track := listenbrainz.Track{
TrackName: recording.Title,
ArtistName: recording.ArtistCredit.String(),
MBIDMapping: &listenbrainz.MBIDMapping{
// In case of redirects this MBID differs from the looked up MBID
RecordingMBID: recording.ID,
ArtistMBIDs: artistMBIDs,
},
}
return &track, nil
}
func AsListen(lbListen listenbrainz.Listen) models.Listen {
listen := models.Listen{
ListenedAt: time.Unix(lbListen.ListenedAt, 0),
UserName: lbListen.UserName,
Track: AsTrack(lbListen.TrackMetadata),
}
return listen
}
func AsLove(f listenbrainz.Feedback) models.Love {
recordingMBID := f.RecordingMBID
track := f.TrackMetadata
if track == nil {
track = &listenbrainz.Track{}
}
love := models.Love{
UserName: f.UserName,
RecordingMBID: recordingMBID,
Created: time.Unix(f.Created, 0),
Track: AsTrack(*track),
}
if love.Track.RecordingMBID == "" {
love.Track.RecordingMBID = love.RecordingMBID
}
return love
}
func AsTrack(t listenbrainz.Track) models.Track {
track := models.Track{
TrackName: t.TrackName,
ReleaseName: t.ReleaseName,
ArtistNames: []string{t.ArtistName},
Duration: t.Duration(),
TrackNumber: t.TrackNumber(),
DiscNumber: t.DiscNumber(),
RecordingMBID: t.RecordingMBID(),
ReleaseMBID: t.ReleaseMBID(),
ReleaseGroupMBID: t.ReleaseGroupMBID(),
ISRC: t.ISRC(),
AdditionalInfo: t.AdditionalInfo,
}
if t.MBIDMapping != nil && len(track.ArtistMBIDs) == 0 {
for _, artistMBID := range t.MBIDMapping.ArtistMBIDs {
track.ArtistMBIDs = append(track.ArtistMBIDs, artistMBID)
}
}
return track
}