feat: parallel downloads (#43)

* feat: parallel downloads

* feat: mod extract progress using file size

* feat: pass mod version in install progress updates

* fix: only close update channels after finished sending

* chore: verbose ci tests

* fix: store mod in cache
chore: add progress logging to tests

* chore: lint

* test: add concurrent download limit

* fix: prevent concurrent map access

* chore: bump pubgrub
fix: fix race conditions

---------

Co-authored-by: Vilsol <me@vil.so>
This commit is contained in:
mircearoata 2023-12-07 00:39:34 +01:00 committed by GitHub
parent a192a63c82
commit 6088d1e8eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 401 additions and 222 deletions

View file

@ -88,6 +88,6 @@ jobs:
run: go generate -tags tools -x ./... run: go generate -tags tools -x ./...
- name: Test - name: Test
run: go test ./... run: go test -v ./...
env: env:
SF_DEDICATED_SERVER: ${{ github.workspace }}/SatisfactoryDedicatedServer SF_DEDICATED_SERVER: ${{ github.workspace }}/SatisfactoryDedicatedServer

View file

@ -17,4 +17,5 @@ func SetDefaults() {
viper.SetDefault("dry-run", false) viper.SetDefault("dry-run", false)
viper.SetDefault("api-base", "https://api.ficsit.app") viper.SetDefault("api-base", "https://api.ficsit.app")
viper.SetDefault("graphql-api", "/v2/query") viper.SetDefault("graphql-api", "/v2/query")
viper.SetDefault("concurrent-downloads", 5)
} }

51
cli/cache/download.go vendored
View file

@ -13,34 +13,11 @@ import (
"github.com/satisfactorymodding/ficsit-cli/utils" "github.com/satisfactorymodding/ficsit-cli/utils"
) )
type Progresser struct { func DownloadOrCache(cacheKey string, hash string, url string, updates chan<- utils.GenericProgress, downloadSemaphore chan int) (io.ReaderAt, int64, error) {
io.Reader if updates != nil {
updates chan utils.GenericUpdate defer close(updates)
total int64
running int64
}
func (pt *Progresser) Read(p []byte) (int, error) {
n, err := pt.Reader.Read(p)
pt.running += int64(n)
if err == nil {
if pt.updates != nil {
select {
case pt.updates <- utils.GenericUpdate{Progress: float64(pt.running) / float64(pt.total)}:
default:
}
}
} }
if err == io.EOF {
return n, io.EOF
}
return n, errors.Wrap(err, "failed to read")
}
func DownloadOrCache(cacheKey string, hash string, url string, updates chan utils.GenericUpdate) (io.ReaderAt, int64, error) {
downloadCache := filepath.Join(viper.GetString("cache-dir"), "downloadCache") downloadCache := filepath.Join(viper.GetString("cache-dir"), "downloadCache")
if err := os.MkdirAll(downloadCache, 0o777); err != nil { if err := os.MkdirAll(downloadCache, 0o777); err != nil {
if !os.IsExist(err) { if !os.IsExist(err) {
@ -82,6 +59,20 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
return nil, 0, errors.Wrap(err, "failed to stat file: "+location) return nil, 0, errors.Wrap(err, "failed to stat file: "+location)
} }
if updates != nil {
headResp, err := http.Head(url)
if err != nil {
return nil, 0, errors.Wrap(err, "failed to head: "+url)
}
defer headResp.Body.Close()
updates <- utils.GenericProgress{Total: headResp.ContentLength}
}
if downloadSemaphore != nil {
downloadSemaphore <- 1
defer func() { <-downloadSemaphore }()
}
out, err := os.Create(location) out, err := os.Create(location)
if err != nil { if err != nil {
return nil, 0, errors.Wrap(err, "failed creating file at: "+location) return nil, 0, errors.Wrap(err, "failed creating file at: "+location)
@ -98,10 +89,10 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
return nil, 0, fmt.Errorf("bad status: %s on url: %s", resp.Status, url) return nil, 0, fmt.Errorf("bad status: %s on url: %s", resp.Status, url)
} }
progresser := &Progresser{ progresser := &utils.Progresser{
Reader: resp.Body, Reader: resp.Body,
total: resp.ContentLength, Total: resp.ContentLength,
updates: updates, Updates: updates,
} }
_, err = io.Copy(out, progresser) _, err = io.Copy(out, progresser)
@ -116,7 +107,7 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
if updates != nil { if updates != nil {
select { select {
case updates <- utils.GenericUpdate{Progress: 1}: case updates <- utils.GenericProgress{Completed: resp.ContentLength, Total: resp.ContentLength}:
default: default:
} }
} }

View file

@ -9,6 +9,7 @@ import (
"github.com/mircearoata/pubgrub-go/pubgrub/helpers" "github.com/mircearoata/pubgrub-go/pubgrub/helpers"
"github.com/mircearoata/pubgrub-go/pubgrub/semver" "github.com/mircearoata/pubgrub-go/pubgrub/semver"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/puzpuzpuz/xsync/v3"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/satisfactorymodding/ficsit-cli/cli/provider" "github.com/satisfactorymodding/ficsit-cli/cli/provider"
@ -35,7 +36,7 @@ type ficsitAPISource struct {
provider provider.Provider provider provider.Provider
lockfile *LockFile lockfile *LockFile
toInstall map[string]semver.Constraint toInstall map[string]semver.Constraint
modVersionInfo map[string]ficsit.ModVersionsWithDependenciesResponse modVersionInfo *xsync.MapOf[string, ficsit.ModVersionsWithDependenciesResponse]
gameVersion semver.Version gameVersion semver.Version
smlVersions []ficsit.SMLVersionsSmlVersionsGetSMLVersionsSml_versionsSMLVersion smlVersions []ficsit.SMLVersionsSmlVersionsGetSMLVersionsSml_versionsSMLVersion
} }
@ -74,7 +75,7 @@ func (f *ficsitAPISource) GetPackageVersions(pkg string) ([]pubgrub.PackageVersi
if response.Mod.Id == "" { if response.Mod.Id == "" {
return nil, errors.Errorf("mod %s not found", pkg) return nil, errors.Errorf("mod %s not found", pkg)
} }
f.modVersionInfo[pkg] = *response f.modVersionInfo.Store(pkg, *response)
versions := make([]pubgrub.PackageVersion, len(response.Mod.Versions)) versions := make([]pubgrub.PackageVersion, len(response.Mod.Versions))
for i, modVersion := range response.Mod.Versions { for i, modVersion := range response.Mod.Versions {
v, err := semver.NewVersion(modVersion.Version) v, err := semver.NewVersion(modVersion.Version)
@ -145,7 +146,7 @@ func (d DependencyResolver) ResolveModDependencies(constraints map[string]string
gameVersion: gameVersionSemver, gameVersion: gameVersionSemver,
lockfile: lockFile, lockfile: lockFile,
toInstall: toInstall, toInstall: toInstall,
modVersionInfo: make(map[string]ficsit.ModVersionsWithDependenciesResponse), modVersionInfo: xsync.NewMapOf[string, ficsit.ModVersionsWithDependenciesResponse](),
} }
result, err := pubgrub.Solve(helpers.NewCachingSource(ficsitSource), rootPkg) result, err := pubgrub.Solve(helpers.NewCachingSource(ficsitSource), rootPkg)
@ -170,7 +171,8 @@ func (d DependencyResolver) ResolveModDependencies(constraints map[string]string
} }
continue continue
} }
versions := ficsitSource.modVersionInfo[k].Mod.Versions value, _ := ficsitSource.modVersionInfo.Load(k)
versions := value.Mod.Versions
for _, ver := range versions { for _, ver := range versions {
if ver.Version == v.RawString() { if ver.Version == v.RawString() {
outputLock[k] = LockedMod{ outputLock[k] = LockedMod{

View file

@ -13,6 +13,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/spf13/viper" "github.com/spf13/viper"
"golang.org/x/sync/errgroup"
"github.com/satisfactorymodding/ficsit-cli/cli/cache" "github.com/satisfactorymodding/ficsit-cli/cli/cache"
"github.com/satisfactorymodding/ficsit-cli/cli/disk" "github.com/satisfactorymodding/ficsit-cli/cli/disk"
@ -351,14 +352,27 @@ func (i *Installation) ResolveProfile(ctx *GlobalContext) (LockFile, error) {
return lockfile, nil return lockfile, nil
} }
type InstallUpdateType string
var (
InstallUpdateTypeOverall InstallUpdateType = "overall"
InstallUpdateTypeModDownload InstallUpdateType = "download"
InstallUpdateTypeModExtract InstallUpdateType = "extract"
InstallUpdateTypeModComplete InstallUpdateType = "complete"
)
type InstallUpdate struct { type InstallUpdate struct {
ModName string Type InstallUpdateType
OverallProgress float64 Item InstallUpdateItem
DownloadProgress float64 Progress utils.GenericProgress
ExtractProgress float64
} }
func (i *Installation) Install(ctx *GlobalContext, updates chan InstallUpdate) error { type InstallUpdateItem struct {
Mod string
Version string
}
func (i *Installation) Install(ctx *GlobalContext, updates chan<- InstallUpdate) error {
if err := i.Validate(ctx); err != nil { if err := i.Validate(ctx); err != nil {
return errors.Wrap(err, "failed to validate installation") return errors.Wrap(err, "failed to validate installation")
} }
@ -403,78 +417,65 @@ func (i *Installation) Install(ctx *GlobalContext, updates chan InstallUpdate) e
} }
} }
downloading := true log.Info().Int("concurrency", viper.GetInt("concurrent-downloads")).Str("path", i.Path).Msg("starting installation")
completed := 0
var genericUpdates chan utils.GenericUpdate errg := errgroup.Group{}
channelUsers := sync.WaitGroup{}
downloadSemaphore := make(chan int, viper.GetInt("concurrent-downloads"))
defer close(downloadSemaphore)
var modComplete chan int
if updates != nil { if updates != nil {
var wg sync.WaitGroup channelUsers.Add(1)
wg.Add(1) modComplete = make(chan int)
defer wg.Wait() defer close(modComplete)
genericUpdates = make(chan utils.GenericUpdate)
defer close(genericUpdates)
go func() { go func() {
defer wg.Done() defer channelUsers.Done()
completed := 0
update := InstallUpdate{ for range modComplete {
OverallProgress: float64(completed) / float64(len(lockfile)), completed++
DownloadProgress: 0, overallUpdate := InstallUpdate{
ExtractProgress: 0, Type: InstallUpdateTypeOverall,
} Progress: utils.GenericProgress{
Completed: int64(completed),
select { Total: int64(len(lockfile)),
case updates <- update: },
default:
}
for up := range genericUpdates {
if downloading {
update.DownloadProgress = up.Progress
} else {
update.DownloadProgress = 1
update.ExtractProgress = up.Progress
}
if up.ModReference != nil {
update.ModName = *up.ModReference
}
update.OverallProgress = float64(completed) / float64(len(lockfile))
select {
case updates <- update:
default:
} }
updates <- overallUpdate
} }
}() }()
} }
for modReference, version := range lockfile { for modReference, version := range lockfile {
// Only install if a link is provided, otherwise assume mod is already installed channelUsers.Add(1)
if version.Link != "" { modReference := modReference
downloading = true version := version
errg.Go(func() error {
if genericUpdates != nil { defer channelUsers.Done()
genericUpdates <- utils.GenericUpdate{ModReference: &modReference} // Only install if a link is provided, otherwise assume mod is already installed
if version.Link != "" {
err := downloadAndExtractMod(modReference, version.Version, version.Link, version.Hash, modsDirectory, updates, downloadSemaphore, d)
if err != nil {
return errors.Wrapf(err, "failed to install %s@%s", modReference, version.Version)
}
} }
log.Info().Str("mod_reference", modReference).Str("version", version.Version).Str("link", version.Link).Msg("downloading mod") if modComplete != nil {
reader, size, err := cache.DownloadOrCache(modReference+"_"+version.Version+".zip", version.Hash, version.Link, genericUpdates) modComplete <- 1
if err != nil {
return errors.Wrap(err, "failed to download "+modReference+" from: "+version.Link)
} }
return nil
})
}
downloading = false if updates != nil {
go func() {
channelUsers.Wait()
close(updates)
}()
}
log.Info().Str("mod_reference", modReference).Str("version", version.Version).Str("link", version.Link).Msg("extracting mod") if err := errg.Wait(); err != nil {
if err := utils.ExtractMod(reader, size, filepath.Join(modsDirectory, modReference), version.Hash, genericUpdates, d); err != nil { return errors.Wrap(err, "failed to install mods")
return errors.Wrap(err, "could not extract "+modReference)
}
}
completed++
} }
return nil return nil
@ -518,6 +519,84 @@ func (i *Installation) UpdateMods(ctx *GlobalContext, mods []string) error {
return nil return nil
} }
func downloadAndExtractMod(modReference string, version string, link string, hash string, modsDirectory string, updates chan<- InstallUpdate, downloadSemaphore chan int, d disk.Disk) error {
var downloadUpdates chan utils.GenericProgress
if updates != nil {
// Forward the inner updates as InstallUpdates
downloadUpdates = make(chan utils.GenericProgress)
go func() {
for up := range downloadUpdates {
updates <- InstallUpdate{
Item: InstallUpdateItem{
Mod: modReference,
Version: version,
},
Type: InstallUpdateTypeModDownload,
Progress: up,
}
}
}()
}
log.Info().Str("mod_reference", modReference).Str("version", version).Str("link", link).Msg("downloading mod")
reader, size, err := cache.DownloadOrCache(modReference+"_"+version+".zip", hash, link, downloadUpdates, downloadSemaphore)
if err != nil {
return errors.Wrap(err, "failed to download "+modReference+" from: "+link)
}
var extractUpdates chan utils.GenericProgress
var wg sync.WaitGroup
if updates != nil {
// Forward the inner updates as InstallUpdates
extractUpdates = make(chan utils.GenericProgress)
wg.Add(1)
go func() {
defer wg.Done()
for up := range extractUpdates {
select {
case updates <- InstallUpdate{
Item: InstallUpdateItem{
Mod: modReference,
Version: version,
},
Type: InstallUpdateTypeModExtract,
Progress: up,
}:
default:
}
}
}()
}
log.Info().Str("mod_reference", modReference).Str("version", version).Str("link", link).Msg("extracting mod")
if err := utils.ExtractMod(reader, size, filepath.Join(modsDirectory, modReference), hash, extractUpdates, d); err != nil {
return errors.Wrap(err, "could not extract "+modReference)
}
if updates != nil {
select {
case updates <- InstallUpdate{
Type: InstallUpdateTypeModComplete,
Item: InstallUpdateItem{
Mod: modReference,
Version: version,
},
}:
default:
}
close(extractUpdates)
}
wg.Wait()
return nil
}
func (i *Installation) SetProfile(ctx *GlobalContext, profile string) error { func (i *Installation) SetProfile(ctx *GlobalContext, profile string) error {
found := false found := false
for _, p := range ctx.Profiles.Profiles { for _, p := range ctx.Profiles.Profiles {

View file

@ -35,11 +35,11 @@ func TestAddInstallation(t *testing.T) {
testza.AssertNoError(t, err) testza.AssertNoError(t, err)
testza.AssertNotNil(t, installation) testza.AssertNotNil(t, installation)
err = installation.Install(ctx, nil) err = installation.Install(ctx, installWatcher())
testza.AssertNoError(t, err) testza.AssertNoError(t, err)
installation.Vanilla = true installation.Vanilla = true
err = installation.Install(ctx, nil) err = installation.Install(ctx, installWatcher())
testza.AssertNoError(t, err) testza.AssertNoError(t, err)
} }
} }

View file

@ -6,8 +6,15 @@ import (
"testing" "testing"
"github.com/MarvinJWendt/testza" "github.com/MarvinJWendt/testza"
"github.com/rs/zerolog/log"
"github.com/satisfactorymodding/ficsit-cli/cfg"
) )
func init() {
cfg.SetDefaults()
}
func profilesGetResolver() DependencyResolver { func profilesGetResolver() DependencyResolver {
ctx, err := InitCLI(false) ctx, err := InitCLI(false)
if err != nil { if err != nil {
@ -17,6 +24,22 @@ func profilesGetResolver() DependencyResolver {
return NewDependencyResolver(ctx.Provider) return NewDependencyResolver(ctx.Provider)
} }
func installWatcher() chan<- InstallUpdate {
c := make(chan InstallUpdate)
go func() {
for i := range c {
if i.Progress.Total == i.Progress.Completed {
if i.Type != InstallUpdateTypeOverall {
log.Info().Str("mod_reference", i.Item.Mod).Str("version", i.Item.Version).Str("type", string(i.Type)).Msg("progress completed")
} else {
log.Info().Msg("overall completed")
}
}
}
}()
return c
}
func TestProfileResolution(t *testing.T) { func TestProfileResolution(t *testing.T) {
resolver := profilesGetResolver() resolver := profilesGetResolver()
@ -108,7 +131,7 @@ func TestUpdateMods(t *testing.T) {
err = installation.WriteLockFile(ctx, oldLockfile) err = installation.WriteLockFile(ctx, oldLockfile)
testza.AssertNoError(t, err) testza.AssertNoError(t, err)
err = installation.Install(ctx, nil) err = installation.Install(ctx, installWatcher())
testza.AssertNoError(t, err) testza.AssertNoError(t, err)
lockFile, err := installation.LockFile(ctx) lockFile, err := installation.LockFile(ctx)
@ -126,7 +149,7 @@ func TestUpdateMods(t *testing.T) {
testza.AssertEqual(t, 2, len(*lockFile)) testza.AssertEqual(t, 2, len(*lockFile))
testza.AssertEqual(t, "1.6.6", (*lockFile)["AreaActions"].Version) testza.AssertEqual(t, "1.6.6", (*lockFile)["AreaActions"].Version)
err = installation.Install(ctx, nil) err = installation.Install(ctx, installWatcher())
testza.AssertNoError(t, err) testza.AssertNoError(t, err)
} }
} }

View file

@ -139,6 +139,7 @@ func init() {
RootCmd.PersistentFlags().String("api-key", "", "API key to use when sending requests") RootCmd.PersistentFlags().String("api-key", "", "API key to use when sending requests")
RootCmd.PersistentFlags().Bool("offline", false, "Whether to only use local data") RootCmd.PersistentFlags().Bool("offline", false, "Whether to only use local data")
RootCmd.PersistentFlags().Int("concurrent-downloads", 5, "Maximum number of concurrent downloads")
_ = viper.BindPFlag("log", RootCmd.PersistentFlags().Lookup("log")) _ = viper.BindPFlag("log", RootCmd.PersistentFlags().Lookup("log"))
_ = viper.BindPFlag("log-file", RootCmd.PersistentFlags().Lookup("log-file")) _ = viper.BindPFlag("log-file", RootCmd.PersistentFlags().Lookup("log-file"))
@ -157,4 +158,5 @@ func init() {
_ = viper.BindPFlag("api-key", RootCmd.PersistentFlags().Lookup("api-key")) _ = viper.BindPFlag("api-key", RootCmd.PersistentFlags().Lookup("api-key"))
_ = viper.BindPFlag("offline", RootCmd.PersistentFlags().Lookup("offline")) _ = viper.BindPFlag("offline", RootCmd.PersistentFlags().Lookup("offline"))
_ = viper.BindPFlag("concurrent-downloads", RootCmd.PersistentFlags().Lookup("concurrent-downloads"))
} }

4
go.mod
View file

@ -13,14 +13,16 @@ require (
github.com/charmbracelet/glamour v0.5.0 github.com/charmbracelet/glamour v0.5.0
github.com/charmbracelet/lipgloss v0.6.0 github.com/charmbracelet/lipgloss v0.6.0
github.com/jlaffaye/ftp v0.1.0 github.com/jlaffaye/ftp v0.1.0
github.com/mircearoata/pubgrub-go v0.3.2 github.com/mircearoata/pubgrub-go v0.3.3
github.com/muesli/reflow v0.3.0 github.com/muesli/reflow v0.3.0
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/pterm/pterm v0.12.67 github.com/pterm/pterm v0.12.67
github.com/puzpuzpuz/xsync/v3 v3.0.2
github.com/rs/zerolog v1.28.0 github.com/rs/zerolog v1.28.0
github.com/sahilm/fuzzy v0.1.0 github.com/sahilm/fuzzy v0.1.0
github.com/spf13/cobra v1.6.0 github.com/spf13/cobra v1.6.0
github.com/spf13/viper v1.13.0 github.com/spf13/viper v1.13.0
golang.org/x/sync v0.1.0
) )
require ( require (

7
go.sum
View file

@ -269,8 +269,8 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
github.com/mircearoata/pubgrub-go v0.3.2 h1:AeC2bvHebii6YrfyN+AST06WiQUENKQkREA7Xoi4/HY= github.com/mircearoata/pubgrub-go v0.3.3 h1:XGwL8Xh5GX+mbnvWItbM/lVJxAq3NZtfUtbJ/hUf2ig=
github.com/mircearoata/pubgrub-go v0.3.2/go.mod h1:9oWL9ZXdjFYvnGl95qiM1dTciFNx1MN8fUnG3SUwDi8= github.com/mircearoata/pubgrub-go v0.3.3/go.mod h1:9oWL9ZXdjFYvnGl95qiM1dTciFNx1MN8fUnG3SUwDi8=
github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@ -310,6 +310,8 @@ github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5b
github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s= github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s=
github.com/pterm/pterm v0.12.67 h1:5iB7ajIQROYfxYD7+sFJ4+KJhFJ+xn7QOVBm4s6RUF0= github.com/pterm/pterm v0.12.67 h1:5iB7ajIQROYfxYD7+sFJ4+KJhFJ+xn7QOVBm4s6RUF0=
github.com/pterm/pterm v0.12.67/go.mod h1:nFuT9ZVkkCi8o4L1dtWuYPwDQxggLh4C263qG5nTLpQ= github.com/pterm/pterm v0.12.67/go.mod h1:nFuT9ZVkkCi8o4L1dtWuYPwDQxggLh4C263qG5nTLpQ=
github.com/puzpuzpuz/xsync/v3 v3.0.2 h1:3yESHrRFYr6xzkz61LLkvNiPFXxJEAABanTQpKbAaew=
github.com/puzpuzpuz/xsync/v3 v3.0.2/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
@ -493,6 +495,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

View file

@ -1,6 +1,8 @@
package scenes package scenes
import ( import (
"sort"
"github.com/charmbracelet/bubbles/progress" "github.com/charmbracelet/bubbles/progress"
tea "github.com/charmbracelet/bubbletea" tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss" "github.com/charmbracelet/lipgloss"
@ -9,120 +11,86 @@ import (
"github.com/satisfactorymodding/ficsit-cli/cli" "github.com/satisfactorymodding/ficsit-cli/cli"
"github.com/satisfactorymodding/ficsit-cli/tea/components" "github.com/satisfactorymodding/ficsit-cli/tea/components"
"github.com/satisfactorymodding/ficsit-cli/tea/scenes/keys" "github.com/satisfactorymodding/ficsit-cli/tea/scenes/keys"
"github.com/satisfactorymodding/ficsit-cli/tea/utils" teaUtils "github.com/satisfactorymodding/ficsit-cli/tea/utils"
"github.com/satisfactorymodding/ficsit-cli/utils"
) )
var _ tea.Model = (*apply)(nil) var _ tea.Model = (*apply)(nil)
type update struct { type modProgress struct {
installName string downloadProgress utils.GenericProgress
modName string extractProgress utils.GenericProgress
completed []string downloading bool
installTotal int complete bool
installCurrent int }
modTotal int
modCurrent int type status struct {
done bool modProgresses map[string]modProgress
installName string
overallProgress utils.GenericProgress
done bool
} }
type apply struct { type apply struct {
root components.RootModel root components.RootModel
parent tea.Model parent tea.Model
error *components.ErrorComponent error *components.ErrorComponent
updateChannel chan update installChannel chan string
errorChannel chan error updateChannel chan cli.InstallUpdate
cancelChannel chan bool doneChannel chan bool
title string errorChannel chan error
status update cancelChannel chan bool
overall progress.Model title string
sub progress.Model status status
cancelled bool overall progress.Model
sub progress.Model
cancelled bool
} }
func NewApply(root components.RootModel, parent tea.Model) tea.Model { func NewApply(root components.RootModel, parent tea.Model) tea.Model {
overall := progress.New(progress.WithSolidFill("118")) overall := progress.New(progress.WithSolidFill("118"))
sub := progress.New(progress.WithSolidFill("202")) sub := progress.New(progress.WithSolidFill("202"))
updateChannel := make(chan update) installChannel := make(chan string)
updateChannel := make(chan cli.InstallUpdate)
doneChannel := make(chan bool, 1)
errorChannel := make(chan error) errorChannel := make(chan error)
cancelChannel := make(chan bool, 1) cancelChannel := make(chan bool, 1)
model := &apply{ model := &apply{
root: root, root: root,
parent: parent, parent: parent,
title: utils.NonListTitleStyle.MarginTop(1).MarginBottom(1).Render("Applying Changes"), title: teaUtils.NonListTitleStyle.MarginTop(1).MarginBottom(1).Render("Applying Changes"),
overall: overall, overall: overall,
sub: sub, sub: sub,
status: update{ status: status{
completed: []string{}, installName: "",
done: false,
installName: "",
installTotal: 100,
installCurrent: 0,
modName: "",
modTotal: 100,
modCurrent: 0,
done: false,
}, },
updateChannel: updateChannel, installChannel: installChannel,
errorChannel: errorChannel, updateChannel: updateChannel,
cancelChannel: cancelChannel, doneChannel: doneChannel,
cancelled: false, errorChannel: errorChannel,
cancelChannel: cancelChannel,
cancelled: false,
} }
go func() { go func() {
result := &update{
completed: make([]string, 0),
installName: "",
installTotal: 100,
installCurrent: 0,
modName: "",
modTotal: 100,
modCurrent: 0,
done: false,
}
updateChannel <- *result
for _, installation := range root.GetGlobal().Installations.Installations { for _, installation := range root.GetGlobal().Installations.Installations {
result.installName = installation.Path installChannel <- installation.Path
updateChannel <- *result
installChannel := make(chan cli.InstallUpdate)
installUpdateChannel := make(chan cli.InstallUpdate)
go func() { go func() {
for data := range installChannel { for update := range installUpdateChannel {
result.installName = installation.Path updateChannel <- update
result.installCurrent = int(data.OverallProgress * 100)
if data.DownloadProgress < 1 {
result.modName = "Downloading: " + data.ModName
result.modCurrent = int(data.DownloadProgress * 100)
} else {
result.modName = "Extracting: " + data.ModName
result.modCurrent = int(data.ExtractProgress * 100)
}
updateChannel <- *result
} }
}() }()
if err := installation.Install(root.GetGlobal(), installChannel); err != nil { if err := installation.Install(root.GetGlobal(), installUpdateChannel); err != nil {
errorChannel <- err errorChannel <- err
return return
} }
close(installChannel)
result.modName = ""
result.installTotal = 100
result.completed = append(result.completed, installation.Path)
updateChannel <- *result
stop := false stop := false
select { select {
case <-cancelChannel: case <-cancelChannel:
@ -135,16 +103,14 @@ func NewApply(root components.RootModel, parent tea.Model) tea.Model {
} }
} }
result.done = true doneChannel <- true
result.installName = ""
updateChannel <- *result
}() }()
return model return model
} }
func (m apply) Init() tea.Cmd { func (m apply) Init() tea.Cmd {
return utils.Ticker() return teaUtils.Ticker()
} }
func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) { func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
@ -169,10 +135,38 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.root.SetSize(msg) m.root.SetSize(msg)
case components.ErrorComponentTimeoutMsg: case components.ErrorComponentTimeoutMsg:
m.error = nil m.error = nil
case utils.TickMsg: case teaUtils.TickMsg:
select { select {
case newStatus := <-m.updateChannel: case <-m.doneChannel:
m.status = newStatus m.status.done = true
m.status.installName = ""
break
case installName := <-m.installChannel:
m.status.installName = installName
m.status.modProgresses = make(map[string]modProgress)
m.status.overallProgress = utils.GenericProgress{}
break
case update := <-m.updateChannel:
switch update.Type {
case cli.InstallUpdateTypeOverall:
m.status.overallProgress = update.Progress
case cli.InstallUpdateTypeModDownload:
m.status.modProgresses[update.Item.Mod] = modProgress{
downloadProgress: update.Progress,
downloading: true,
complete: false,
}
case cli.InstallUpdateTypeModExtract:
m.status.modProgresses[update.Item.Mod] = modProgress{
extractProgress: update.Progress,
downloading: false,
complete: false,
}
case cli.InstallUpdateTypeModComplete:
m.status.modProgresses[update.Item.Mod] = modProgress{
complete: true,
}
}
break break
case err := <-m.errorChannel: case err := <-m.errorChannel:
wrappedErrMessage := wrap.String(err.Error(), int(float64(m.root.Size().Width)*0.8)) wrappedErrMessage := wrap.String(err.Error(), int(float64(m.root.Size().Width)*0.8))
@ -183,7 +177,7 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
// Skip if nothing there // Skip if nothing there
break break
} }
return m, utils.Ticker() return m, teaUtils.Ticker()
} }
return m, nil return m, nil
@ -191,30 +185,38 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
func (m apply) View() string { func (m apply) View() string {
strs := make([]string, 0) strs := make([]string, 0)
for _, s := range m.status.completed {
strs = append(strs, lipgloss.NewStyle().Foreground(lipgloss.Color("22")).Render("✓ ")+s)
}
if m.status.installName != "" { if m.status.installName != "" {
marginTop := 0 strs = append(strs, lipgloss.NewStyle().Render(m.status.installName))
if len(m.status.completed) > 0 { strs = append(strs, lipgloss.NewStyle().MarginBottom(1).Render(m.overall.ViewAs(m.status.overallProgress.Percentage())))
marginTop = 1
}
strs = append(strs, lipgloss.NewStyle().MarginTop(marginTop).Render(m.status.installName))
strs = append(strs, m.overall.ViewAs(float64(m.status.installCurrent)/float64(m.status.installTotal)))
} }
if m.status.modName != "" { keys := make([]string, 0)
strs = append(strs, lipgloss.NewStyle().MarginTop(1).Render(m.status.modName)) for k := range m.status.modProgresses {
strs = append(strs, m.sub.ViewAs(float64(m.status.modCurrent)/float64(m.status.modTotal))) keys = append(keys, k)
}
sort.Strings(keys)
for _, modReference := range keys {
p := m.status.modProgresses[modReference]
if p.complete {
strs = append(strs, lipgloss.NewStyle().Foreground(lipgloss.Color("22")).Render("✓ ")+modReference)
} else {
if p.downloading {
strs = append(strs, lipgloss.NewStyle().Render(modReference+" (Downloading)"))
strs = append(strs, m.sub.ViewAs(p.downloadProgress.Percentage()))
} else {
strs = append(strs, lipgloss.NewStyle().Render(modReference+" (Extracting)"))
strs = append(strs, m.sub.ViewAs(p.extractProgress.Percentage()))
}
}
} }
if m.status.done { if m.status.done {
if m.cancelled { if m.cancelled {
strs = append(strs, utils.LabelStyle.Copy().Foreground(lipgloss.Color("196")).Padding(0).Margin(1).Render("Cancelled! Press Enter to return")) strs = append(strs, teaUtils.LabelStyle.Copy().Foreground(lipgloss.Color("196")).Padding(0).Margin(1).Render("Cancelled! Press Enter to return"))
} else { } else {
strs = append(strs, utils.LabelStyle.Copy().Padding(0).Margin(1).Render("Done! Press Enter to return")) strs = append(strs, teaUtils.LabelStyle.Copy().Padding(0).Margin(1).Render("Done! Press Enter to return"))
} }
} }

View file

@ -7,15 +7,51 @@ import (
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
"sync"
"sync/atomic"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/satisfactorymodding/ficsit-cli/cli/disk" "github.com/satisfactorymodding/ficsit-cli/cli/disk"
) )
type GenericUpdate struct { type GenericProgress struct {
ModReference *string Completed int64
Progress float64 Total int64
}
func (gp GenericProgress) Percentage() float64 {
if gp.Total == 0 {
return 0
}
return float64(gp.Completed) / float64(gp.Total)
}
type Progresser struct {
io.Reader
Updates chan<- GenericProgress
Total int64
Running int64
}
func (pt *Progresser) Read(p []byte) (int, error) {
n, err := pt.Reader.Read(p)
pt.Running += int64(n)
if err == nil {
if pt.Updates != nil {
select {
case pt.Updates <- GenericProgress{Completed: pt.Running, Total: pt.Total}:
default:
}
}
}
if err == io.EOF {
return n, io.EOF
}
return n, errors.Wrap(err, "failed to read")
} }
func SHA256Data(f io.Reader) (string, error) { func SHA256Data(f io.Reader) (string, error) {
@ -27,7 +63,7 @@ func SHA256Data(f io.Reader) (string, error) {
return hex.EncodeToString(h.Sum(nil)), nil return hex.EncodeToString(h.Sum(nil)), nil
} }
func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates chan GenericUpdate, d disk.Disk) error { func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates chan<- GenericProgress, d disk.Disk) error {
hashFile := filepath.Join(location, ".smm") hashFile := filepath.Join(location, ".smm")
hashBytes, err := d.Read(hashFile) hashBytes, err := d.Read(hashFile)
if err != nil { if err != nil {
@ -59,7 +95,24 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
return errors.Wrap(err, "failed to read file as zip") return errors.Wrap(err, "failed to read file as zip")
} }
for i, file := range reader.File { totalSize := int64(0)
for _, file := range reader.File {
totalSize += int64(file.UncompressedSize64)
}
totalExtracted := int64(0)
totalExtractedPtr := &totalExtracted
channelUsers := sync.WaitGroup{}
if updates != nil {
defer func() {
channelUsers.Wait()
}()
}
for _, file := range reader.File {
if !file.FileInfo().IsDir() { if !file.FileInfo().IsDir() {
outFileLocation := filepath.Join(location, file.Name) outFileLocation := filepath.Join(location, file.Name)
@ -67,16 +120,26 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
return errors.Wrap(err, "failed to create mod directory: "+location) return errors.Wrap(err, "failed to create mod directory: "+location)
} }
if err := writeZipFile(outFileLocation, file, d); err != nil { var fileUpdates chan GenericProgress
if updates != nil {
fileUpdates = make(chan GenericProgress)
channelUsers.Add(1)
go func() {
defer channelUsers.Done()
for fileUpdate := range fileUpdates {
updates <- GenericProgress{
Completed: atomic.LoadInt64(totalExtractedPtr) + fileUpdate.Completed,
Total: totalSize,
}
}
}()
}
if err := writeZipFile(outFileLocation, file, d, fileUpdates); err != nil {
return err return err
} }
}
if updates != nil { atomic.AddInt64(totalExtractedPtr, int64(file.UncompressedSize64))
select {
case updates <- GenericUpdate{Progress: float64(i) / float64(len(reader.File)-1)}:
default:
}
} }
} }
@ -86,7 +149,7 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
if updates != nil { if updates != nil {
select { select {
case updates <- GenericUpdate{Progress: 1}: case updates <- GenericProgress{Completed: totalSize, Total: totalSize}:
default: default:
} }
} }
@ -94,7 +157,11 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
return nil return nil
} }
func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk) error { func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk, updates chan<- GenericProgress) error {
if updates != nil {
defer close(updates)
}
outFile, err := d.Open(outFileLocation, os.O_CREATE|os.O_RDWR) outFile, err := d.Open(outFileLocation, os.O_CREATE|os.O_RDWR)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to write to file: "+outFileLocation) return errors.Wrap(err, "failed to write to file: "+outFileLocation)
@ -106,8 +173,15 @@ func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk) error {
if err != nil { if err != nil {
return errors.Wrap(err, "failed to process mod zip") return errors.Wrap(err, "failed to process mod zip")
} }
defer inFile.Close()
if _, err := io.Copy(outFile, inFile); err != nil { progressInReader := &Progresser{
Reader: inFile,
Total: int64(file.UncompressedSize64),
Updates: updates,
}
if _, err := io.Copy(outFile, progressInReader); err != nil {
return errors.Wrap(err, "failed to write to file: "+outFileLocation) return errors.Wrap(err, "failed to write to file: "+outFileLocation)
} }