mirror of
https://git.sr.ht/~phw/scotty
synced 2025-04-25 05:47:57 +02:00
Transfer command refactoring
Make transfer command processing even more modular
This commit is contained in:
Philipp Wolfer
parent
7c85ba05ab
commit
a9e07915ce
7 changed files with 236 additions and 179 deletions
|
|
@ -25,6 +25,7 @@ import (
|
|||
"github.com/spf13/viper"
|
||||
"go.uploadedlobster.com/scotty/internal/auth"
|
||||
"go.uploadedlobster.com/scotty/internal/backends"
|
||||
"go.uploadedlobster.com/scotty/internal/cli"
|
||||
"go.uploadedlobster.com/scotty/internal/models"
|
||||
"go.uploadedlobster.com/scotty/internal/storage"
|
||||
"golang.org/x/oauth2"
|
||||
|
|
@ -36,7 +37,7 @@ var authCmd = &cobra.Command{
|
|||
Short: "Authenticate with a backend",
|
||||
Long: `For backends requiring authentication this command can be used to authenticate.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
serviceName, serviceConfig := getConfigFromFlag(cmd, "service")
|
||||
serviceName, serviceConfig := cli.GetConfigFromFlag(cmd, "service")
|
||||
backend, err := backends.ResolveBackend[models.OAuth2Authenticator](serviceConfig)
|
||||
cobra.CheckErr(err)
|
||||
|
||||
|
|
|
|||
161
cmd/common.go
161
cmd/common.go
|
|
@ -1,161 +0,0 @@
|
|||
/*
|
||||
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
|
||||
|
||||
This file is part of Scotty.
|
||||
|
||||
Scotty is free software: you can redistribute it and/or modify it under the
|
||||
terms of the GNU General Public License as published by the Free Software
|
||||
Foundation, either version 3 of the License, or (at your option) any later version.
|
||||
|
||||
Scotty is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along with
|
||||
Scotty. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
"go.uploadedlobster.com/scotty/internal/backends"
|
||||
"go.uploadedlobster.com/scotty/internal/models"
|
||||
"go.uploadedlobster.com/scotty/internal/storage"
|
||||
)
|
||||
|
||||
func getConfigFromFlag(cmd *cobra.Command, flagName string) (string, *viper.Viper) {
|
||||
configName := cmd.Flag(flagName).Value.String()
|
||||
var config *viper.Viper
|
||||
servicesConfig := viper.Sub("service")
|
||||
if servicesConfig != nil {
|
||||
config = servicesConfig.Sub(configName)
|
||||
}
|
||||
if config == nil {
|
||||
cobra.CheckErr(fmt.Sprintf("Invalid source configuration \"%s\"", configName))
|
||||
}
|
||||
return configName, config
|
||||
}
|
||||
|
||||
func getInt64FromFlag(cmd *cobra.Command, flagName string) (result int64) {
|
||||
result, err := cmd.Flags().GetInt64(flagName)
|
||||
if err != nil {
|
||||
result = 0
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type backendInfo[T models.Backend, R models.ListensResult | models.LovesResult] struct {
|
||||
configName string
|
||||
backend T
|
||||
}
|
||||
|
||||
type exportBackendInfo[T models.Backend, R models.ListensResult | models.LovesResult] struct {
|
||||
backendInfo[T, R]
|
||||
processor backends.ExportProcessor[R]
|
||||
}
|
||||
|
||||
type importBackendInfo[T models.Backend, R models.ListensResult | models.LovesResult] struct {
|
||||
backendInfo[T, R]
|
||||
processor backends.ImportProcessor[R]
|
||||
}
|
||||
|
||||
func resolveBackends[E models.Backend, I models.ImportBackend, R models.ListensResult | models.LovesResult](cmd *cobra.Command) (*exportBackendInfo[E, R], *importBackendInfo[I, R], error) {
|
||||
sourceName, sourceConfig := getConfigFromFlag(cmd, "from")
|
||||
targetName, targetConfig := getConfigFromFlag(cmd, "to")
|
||||
// Initialize backends
|
||||
exportBackend, err := backends.ResolveBackend[E](sourceConfig)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
importBackend, err := backends.ResolveBackend[I](targetConfig)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
exportInfo := exportBackendInfo[E, R]{
|
||||
backendInfo: backendInfo[E, R]{
|
||||
configName: sourceName,
|
||||
backend: exportBackend,
|
||||
},
|
||||
}
|
||||
|
||||
importInfo := importBackendInfo[I, R]{
|
||||
backendInfo: backendInfo[I, R]{
|
||||
configName: targetName,
|
||||
backend: importBackend,
|
||||
},
|
||||
}
|
||||
|
||||
return &exportInfo, &importInfo, nil
|
||||
}
|
||||
|
||||
func cmdExportImport[E models.Backend, I models.ImportBackend, R models.ListensResult | models.LovesResult](cmd *cobra.Command, entity string, exp *exportBackendInfo[E, R], imp *importBackendInfo[I, R]) {
|
||||
sourceName := exp.configName
|
||||
targetName := imp.configName
|
||||
fmt.Printf("Transferring %s from %s to %s...\n", entity, sourceName, targetName)
|
||||
|
||||
// Setup database
|
||||
db, err := storage.New(viper.GetString("database"))
|
||||
cobra.CheckErr(err)
|
||||
|
||||
// Authenticate backends, if needed
|
||||
config := viper.GetViper()
|
||||
_, err = backends.Authenticate(sourceName, exp.backend, db, config)
|
||||
cobra.CheckErr(err)
|
||||
|
||||
_, err = backends.Authenticate(targetName, imp.backend, db, config)
|
||||
cobra.CheckErr(err)
|
||||
|
||||
// Read timestamp
|
||||
timestamp := time.Unix(getInt64FromFlag(cmd, "timestamp"), 0)
|
||||
if timestamp == time.Unix(0, 0) {
|
||||
timestamp, err = db.GetImportTimestamp(sourceName, targetName, entity)
|
||||
cobra.CheckErr(err)
|
||||
}
|
||||
fmt.Printf("From timestamp: %v (%v)\n", timestamp, timestamp.Unix())
|
||||
|
||||
// Prepare progress bars
|
||||
exportProgress := make(chan models.Progress)
|
||||
importProgress := make(chan models.Progress)
|
||||
var wg sync.WaitGroup
|
||||
progress := progressBar(&wg, exportProgress, importProgress)
|
||||
|
||||
// Export from source
|
||||
exportChan := make(chan R, 1000)
|
||||
go exp.processor.Process(timestamp, exportChan, exportProgress)
|
||||
|
||||
// Import into target
|
||||
resultChan := make(chan models.ImportResult)
|
||||
go imp.processor.Process(exportChan, resultChan, importProgress)
|
||||
result := <-resultChan
|
||||
close(exportProgress)
|
||||
wg.Wait()
|
||||
progress.Wait()
|
||||
if result.Error != nil {
|
||||
fmt.Printf("Import failed, last reported timestamp was %v (%v)\n", result.LastTimestamp, result.LastTimestamp.Unix())
|
||||
cobra.CheckErr(result.Error)
|
||||
}
|
||||
fmt.Printf("Imported %v of %v %s into %v.\n",
|
||||
result.ImportCount, result.TotalCount, entity, targetName)
|
||||
|
||||
// Update timestamp
|
||||
if result.LastTimestamp.Unix() < timestamp.Unix() {
|
||||
result.LastTimestamp = timestamp
|
||||
}
|
||||
fmt.Printf("Latest timestamp: %v (%v)\n", result.LastTimestamp, result.LastTimestamp.Unix())
|
||||
err = db.SetImportTimestamp(sourceName, targetName, entity, result.LastTimestamp)
|
||||
cobra.CheckErr(err)
|
||||
|
||||
// Print errors
|
||||
if len(result.ImportErrors) > 0 {
|
||||
fmt.Printf("\nDuring the import the following errors occurred:\n")
|
||||
for _, err := range result.ImportErrors {
|
||||
fmt.Printf("Error: %v\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -18,8 +18,11 @@ package cmd
|
|||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
"go.uploadedlobster.com/scotty/internal/backends"
|
||||
"go.uploadedlobster.com/scotty/internal/cli"
|
||||
"go.uploadedlobster.com/scotty/internal/models"
|
||||
"go.uploadedlobster.com/scotty/internal/storage"
|
||||
)
|
||||
|
||||
// listensCmd represents the listens command
|
||||
|
|
@ -28,15 +31,18 @@ var listensCmd = &cobra.Command{
|
|||
Short: "Transfer listens between two services",
|
||||
Long: `Transfers listens between two configured services.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
exp, imp, err := resolveBackends[models.ListensExport, models.ListensImport, models.ListensResult](cmd)
|
||||
db, err := storage.New(viper.GetString("database"))
|
||||
cobra.CheckErr(err)
|
||||
c, err := cli.NewTransferCmd[
|
||||
models.ListensExport,
|
||||
models.ListensImport,
|
||||
models.ListensResult,
|
||||
](cmd, &db, "listens")
|
||||
cobra.CheckErr(err)
|
||||
exp := backends.ListensExportProcessor{Backend: c.ExpBackend}
|
||||
imp := backends.ListensImportProcessor{Backend: c.ImpBackend}
|
||||
err = c.Transfer(exp, imp)
|
||||
cobra.CheckErr(err)
|
||||
exp.processor = backends.ListensExportProcessor{
|
||||
Backend: exp.backend,
|
||||
}
|
||||
imp.processor = backends.ListensImportProcessor{
|
||||
Backend: imp.backend,
|
||||
}
|
||||
cmdExportImport(cmd, "listens", exp, imp)
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
22
cmd/loves.go
22
cmd/loves.go
|
|
@ -18,8 +18,11 @@ package cmd
|
|||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
"go.uploadedlobster.com/scotty/internal/backends"
|
||||
"go.uploadedlobster.com/scotty/internal/cli"
|
||||
"go.uploadedlobster.com/scotty/internal/models"
|
||||
"go.uploadedlobster.com/scotty/internal/storage"
|
||||
)
|
||||
|
||||
// lovesCmd represents the loves command
|
||||
|
|
@ -28,15 +31,18 @@ var lovesCmd = &cobra.Command{
|
|||
Short: "Transfer loves between two services",
|
||||
Long: `Transfers loves between two configured services.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
exp, imp, err := resolveBackends[models.LovesExport, models.LovesImport, models.LovesResult](cmd)
|
||||
db, err := storage.New(viper.GetString("database"))
|
||||
cobra.CheckErr(err)
|
||||
c, err := cli.NewTransferCmd[
|
||||
models.LovesExport,
|
||||
models.LovesImport,
|
||||
models.LovesResult,
|
||||
](cmd, &db, "loves")
|
||||
cobra.CheckErr(err)
|
||||
exp := backends.LovesExportProcessor{Backend: c.ExpBackend}
|
||||
imp := backends.LovesImportProcessor{Backend: c.ImpBackend}
|
||||
err = c.Transfer(exp, imp)
|
||||
cobra.CheckErr(err)
|
||||
exp.processor = backends.LovesExportProcessor{
|
||||
Backend: exp.backend,
|
||||
}
|
||||
imp.processor = backends.LovesImportProcessor{
|
||||
Backend: imp.backend,
|
||||
}
|
||||
cmdExportImport(cmd, "loves", exp, imp)
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,79 +0,0 @@
|
|||
/*
|
||||
Copyright © 2023 Philipp Wolfer <phw@uploadedlobster.com>
|
||||
|
||||
This file is part of Scotty.
|
||||
|
||||
Scotty is free software: you can redistribute it and/or modify it under the
|
||||
terms of the GNU General Public License as published by the Free Software
|
||||
Foundation, either version 3 of the License, or (at your option) any later version.
|
||||
|
||||
Scotty is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along with
|
||||
Scotty. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/vbauerster/mpb/v8"
|
||||
"github.com/vbauerster/mpb/v8/decor"
|
||||
"go.uploadedlobster.com/scotty/internal/models"
|
||||
)
|
||||
|
||||
func progressBar(wg *sync.WaitGroup, exportProgress chan models.Progress, importProgress chan models.Progress) *mpb.Progress {
|
||||
p := mpb.New(
|
||||
mpb.WithWaitGroup(wg),
|
||||
mpb.WithOutput(color.Output),
|
||||
// mpb.WithWidth(64),
|
||||
mpb.WithAutoRefresh(),
|
||||
)
|
||||
|
||||
exportBar := setupProgressBar(p, "exporting")
|
||||
importBar := setupProgressBar(p, "importing")
|
||||
go updateProgressBar(exportBar, wg, exportProgress)
|
||||
go updateProgressBar(importBar, wg, importProgress)
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar {
|
||||
green := color.New(color.FgGreen).SprintFunc()
|
||||
return p.New(0,
|
||||
mpb.BarStyle(),
|
||||
mpb.PrependDecorators(
|
||||
decor.Name(" "),
|
||||
decor.OnComplete(
|
||||
decor.Spinner(nil, decor.WC{W: 2, C: decor.DidentRight}),
|
||||
green("✓ "),
|
||||
),
|
||||
decor.Name(name, decor.WCSyncWidthR),
|
||||
),
|
||||
mpb.AppendDecorators(
|
||||
decor.OnComplete(
|
||||
decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth}),
|
||||
"done",
|
||||
),
|
||||
// decor.OnComplete(decor.Percentage(decor.WC{W: 5, C: decor.DSyncWidthR}), "done"),
|
||||
decor.Name(" "),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func updateProgressBar(bar *mpb.Bar, wg *sync.WaitGroup, progressChan chan models.Progress) {
|
||||
wg.Add(1)
|
||||
defer wg.Done()
|
||||
lastIterTime := time.Now()
|
||||
for progress := range progressChan {
|
||||
oldIterTime := lastIterTime
|
||||
lastIterTime = time.Now()
|
||||
bar.EwmaSetCurrent(progress.Elapsed, lastIterTime.Sub(oldIterTime))
|
||||
bar.SetTotal(progress.Total, progress.Completed)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue