diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index b5f5546..47b3bf8 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -88,6 +88,6 @@ jobs: run: go generate -tags tools -x ./... - name: Test - run: go test ./... + run: go test -v ./... env: SF_DEDICATED_SERVER: ${{ github.workspace }}/SatisfactoryDedicatedServer diff --git a/cfg/test_defaults.go b/cfg/test_defaults.go index 1f59f4c..1e71690 100644 --- a/cfg/test_defaults.go +++ b/cfg/test_defaults.go @@ -17,4 +17,5 @@ func SetDefaults() { viper.SetDefault("dry-run", false) viper.SetDefault("api-base", "https://api.ficsit.app") viper.SetDefault("graphql-api", "/v2/query") + viper.SetDefault("concurrent-downloads", 5) } diff --git a/cli/cache/download.go b/cli/cache/download.go index 22376a6..98b2bec 100644 --- a/cli/cache/download.go +++ b/cli/cache/download.go @@ -13,34 +13,11 @@ import ( "github.com/satisfactorymodding/ficsit-cli/utils" ) -type Progresser struct { - io.Reader - updates chan utils.GenericUpdate - total int64 - running int64 -} - -func (pt *Progresser) Read(p []byte) (int, error) { - n, err := pt.Reader.Read(p) - pt.running += int64(n) - - if err == nil { - if pt.updates != nil { - select { - case pt.updates <- utils.GenericUpdate{Progress: float64(pt.running) / float64(pt.total)}: - default: - } - } +func DownloadOrCache(cacheKey string, hash string, url string, updates chan<- utils.GenericProgress, downloadSemaphore chan int) (io.ReaderAt, int64, error) { + if updates != nil { + defer close(updates) } - if err == io.EOF { - return n, io.EOF - } - - return n, errors.Wrap(err, "failed to read") -} - -func DownloadOrCache(cacheKey string, hash string, url string, updates chan utils.GenericUpdate) (io.ReaderAt, int64, error) { downloadCache := filepath.Join(viper.GetString("cache-dir"), "downloadCache") if err := os.MkdirAll(downloadCache, 0o777); err != nil { if !os.IsExist(err) { @@ -82,6 +59,20 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util return nil, 0, errors.Wrap(err, "failed to stat file: "+location) } + if updates != nil { + headResp, err := http.Head(url) + if err != nil { + return nil, 0, errors.Wrap(err, "failed to head: "+url) + } + defer headResp.Body.Close() + updates <- utils.GenericProgress{Total: headResp.ContentLength} + } + + if downloadSemaphore != nil { + downloadSemaphore <- 1 + defer func() { <-downloadSemaphore }() + } + out, err := os.Create(location) if err != nil { return nil, 0, errors.Wrap(err, "failed creating file at: "+location) @@ -98,10 +89,10 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util return nil, 0, fmt.Errorf("bad status: %s on url: %s", resp.Status, url) } - progresser := &Progresser{ + progresser := &utils.Progresser{ Reader: resp.Body, - total: resp.ContentLength, - updates: updates, + Total: resp.ContentLength, + Updates: updates, } _, err = io.Copy(out, progresser) @@ -116,7 +107,7 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util if updates != nil { select { - case updates <- utils.GenericUpdate{Progress: 1}: + case updates <- utils.GenericProgress{Completed: resp.ContentLength, Total: resp.ContentLength}: default: } } diff --git a/cli/dependency_resolver.go b/cli/dependency_resolver.go index 2d15698..e5158d2 100644 --- a/cli/dependency_resolver.go +++ b/cli/dependency_resolver.go @@ -9,6 +9,7 @@ import ( "github.com/mircearoata/pubgrub-go/pubgrub/helpers" "github.com/mircearoata/pubgrub-go/pubgrub/semver" "github.com/pkg/errors" + "github.com/puzpuzpuz/xsync/v3" "github.com/spf13/viper" "github.com/satisfactorymodding/ficsit-cli/cli/provider" @@ -35,7 +36,7 @@ type ficsitAPISource struct { provider provider.Provider lockfile *LockFile toInstall map[string]semver.Constraint - modVersionInfo map[string]ficsit.ModVersionsWithDependenciesResponse + modVersionInfo *xsync.MapOf[string, ficsit.ModVersionsWithDependenciesResponse] gameVersion semver.Version smlVersions []ficsit.SMLVersionsSmlVersionsGetSMLVersionsSml_versionsSMLVersion } @@ -74,7 +75,7 @@ func (f *ficsitAPISource) GetPackageVersions(pkg string) ([]pubgrub.PackageVersi if response.Mod.Id == "" { return nil, errors.Errorf("mod %s not found", pkg) } - f.modVersionInfo[pkg] = *response + f.modVersionInfo.Store(pkg, *response) versions := make([]pubgrub.PackageVersion, len(response.Mod.Versions)) for i, modVersion := range response.Mod.Versions { v, err := semver.NewVersion(modVersion.Version) @@ -145,7 +146,7 @@ func (d DependencyResolver) ResolveModDependencies(constraints map[string]string gameVersion: gameVersionSemver, lockfile: lockFile, toInstall: toInstall, - modVersionInfo: make(map[string]ficsit.ModVersionsWithDependenciesResponse), + modVersionInfo: xsync.NewMapOf[string, ficsit.ModVersionsWithDependenciesResponse](), } result, err := pubgrub.Solve(helpers.NewCachingSource(ficsitSource), rootPkg) @@ -170,7 +171,8 @@ func (d DependencyResolver) ResolveModDependencies(constraints map[string]string } continue } - versions := ficsitSource.modVersionInfo[k].Mod.Versions + value, _ := ficsitSource.modVersionInfo.Load(k) + versions := value.Mod.Versions for _, ver := range versions { if ver.Version == v.RawString() { outputLock[k] = LockedMod{ diff --git a/cli/installations.go b/cli/installations.go index 403beb3..d0fc083 100644 --- a/cli/installations.go +++ b/cli/installations.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/spf13/viper" + "golang.org/x/sync/errgroup" "github.com/satisfactorymodding/ficsit-cli/cli/cache" "github.com/satisfactorymodding/ficsit-cli/cli/disk" @@ -351,14 +352,27 @@ func (i *Installation) ResolveProfile(ctx *GlobalContext) (LockFile, error) { return lockfile, nil } +type InstallUpdateType string + +var ( + InstallUpdateTypeOverall InstallUpdateType = "overall" + InstallUpdateTypeModDownload InstallUpdateType = "download" + InstallUpdateTypeModExtract InstallUpdateType = "extract" + InstallUpdateTypeModComplete InstallUpdateType = "complete" +) + type InstallUpdate struct { - ModName string - OverallProgress float64 - DownloadProgress float64 - ExtractProgress float64 + Type InstallUpdateType + Item InstallUpdateItem + Progress utils.GenericProgress } -func (i *Installation) Install(ctx *GlobalContext, updates chan InstallUpdate) error { +type InstallUpdateItem struct { + Mod string + Version string +} + +func (i *Installation) Install(ctx *GlobalContext, updates chan<- InstallUpdate) error { if err := i.Validate(ctx); err != nil { return errors.Wrap(err, "failed to validate installation") } @@ -403,78 +417,65 @@ func (i *Installation) Install(ctx *GlobalContext, updates chan InstallUpdate) e } } - downloading := true - completed := 0 + log.Info().Int("concurrency", viper.GetInt("concurrent-downloads")).Str("path", i.Path).Msg("starting installation") - var genericUpdates chan utils.GenericUpdate + errg := errgroup.Group{} + channelUsers := sync.WaitGroup{} + downloadSemaphore := make(chan int, viper.GetInt("concurrent-downloads")) + defer close(downloadSemaphore) + + var modComplete chan int if updates != nil { - var wg sync.WaitGroup - wg.Add(1) - defer wg.Wait() - - genericUpdates = make(chan utils.GenericUpdate) - defer close(genericUpdates) - + channelUsers.Add(1) + modComplete = make(chan int) + defer close(modComplete) go func() { - defer wg.Done() - - update := InstallUpdate{ - OverallProgress: float64(completed) / float64(len(lockfile)), - DownloadProgress: 0, - ExtractProgress: 0, - } - - select { - case updates <- update: - default: - } - - for up := range genericUpdates { - if downloading { - update.DownloadProgress = up.Progress - } else { - update.DownloadProgress = 1 - update.ExtractProgress = up.Progress - } - - if up.ModReference != nil { - update.ModName = *up.ModReference - } - - update.OverallProgress = float64(completed) / float64(len(lockfile)) - - select { - case updates <- update: - default: + defer channelUsers.Done() + completed := 0 + for range modComplete { + completed++ + overallUpdate := InstallUpdate{ + Type: InstallUpdateTypeOverall, + Progress: utils.GenericProgress{ + Completed: int64(completed), + Total: int64(len(lockfile)), + }, } + updates <- overallUpdate } }() } for modReference, version := range lockfile { - // Only install if a link is provided, otherwise assume mod is already installed - if version.Link != "" { - downloading = true - - if genericUpdates != nil { - genericUpdates <- utils.GenericUpdate{ModReference: &modReference} + channelUsers.Add(1) + modReference := modReference + version := version + errg.Go(func() error { + defer channelUsers.Done() + // Only install if a link is provided, otherwise assume mod is already installed + if version.Link != "" { + err := downloadAndExtractMod(modReference, version.Version, version.Link, version.Hash, modsDirectory, updates, downloadSemaphore, d) + if err != nil { + return errors.Wrapf(err, "failed to install %s@%s", modReference, version.Version) + } } - log.Info().Str("mod_reference", modReference).Str("version", version.Version).Str("link", version.Link).Msg("downloading mod") - reader, size, err := cache.DownloadOrCache(modReference+"_"+version.Version+".zip", version.Hash, version.Link, genericUpdates) - if err != nil { - return errors.Wrap(err, "failed to download "+modReference+" from: "+version.Link) + if modComplete != nil { + modComplete <- 1 } + return nil + }) + } - downloading = false + if updates != nil { + go func() { + channelUsers.Wait() + close(updates) + }() + } - log.Info().Str("mod_reference", modReference).Str("version", version.Version).Str("link", version.Link).Msg("extracting mod") - if err := utils.ExtractMod(reader, size, filepath.Join(modsDirectory, modReference), version.Hash, genericUpdates, d); err != nil { - return errors.Wrap(err, "could not extract "+modReference) - } - } - - completed++ + if err := errg.Wait(); err != nil { + return errors.Wrap(err, "failed to install mods") } return nil @@ -518,6 +519,84 @@ func (i *Installation) UpdateMods(ctx *GlobalContext, mods []string) error { return nil } +func downloadAndExtractMod(modReference string, version string, link string, hash string, modsDirectory string, updates chan<- InstallUpdate, downloadSemaphore chan int, d disk.Disk) error { + var downloadUpdates chan utils.GenericProgress + + if updates != nil { + // Forward the inner updates as InstallUpdates + downloadUpdates = make(chan utils.GenericProgress) + + go func() { + for up := range downloadUpdates { + updates <- InstallUpdate{ + Item: InstallUpdateItem{ + Mod: modReference, + Version: version, + }, + Type: InstallUpdateTypeModDownload, + Progress: up, + } + } + }() + } + + log.Info().Str("mod_reference", modReference).Str("version", version).Str("link", link).Msg("downloading mod") + reader, size, err := cache.DownloadOrCache(modReference+"_"+version+".zip", hash, link, downloadUpdates, downloadSemaphore) + if err != nil { + return errors.Wrap(err, "failed to download "+modReference+" from: "+link) + } + + var extractUpdates chan utils.GenericProgress + + var wg sync.WaitGroup + if updates != nil { + // Forward the inner updates as InstallUpdates + extractUpdates = make(chan utils.GenericProgress) + + wg.Add(1) + go func() { + defer wg.Done() + for up := range extractUpdates { + select { + case updates <- InstallUpdate{ + Item: InstallUpdateItem{ + Mod: modReference, + Version: version, + }, + Type: InstallUpdateTypeModExtract, + Progress: up, + }: + default: + } + } + }() + } + + log.Info().Str("mod_reference", modReference).Str("version", version).Str("link", link).Msg("extracting mod") + if err := utils.ExtractMod(reader, size, filepath.Join(modsDirectory, modReference), hash, extractUpdates, d); err != nil { + return errors.Wrap(err, "could not extract "+modReference) + } + + if updates != nil { + select { + case updates <- InstallUpdate{ + Type: InstallUpdateTypeModComplete, + Item: InstallUpdateItem{ + Mod: modReference, + Version: version, + }, + }: + default: + } + + close(extractUpdates) + } + + wg.Wait() + + return nil +} + func (i *Installation) SetProfile(ctx *GlobalContext, profile string) error { found := false for _, p := range ctx.Profiles.Profiles { diff --git a/cli/installations_test.go b/cli/installations_test.go index a60396c..b0e0f28 100644 --- a/cli/installations_test.go +++ b/cli/installations_test.go @@ -35,11 +35,11 @@ func TestAddInstallation(t *testing.T) { testza.AssertNoError(t, err) testza.AssertNotNil(t, installation) - err = installation.Install(ctx, nil) + err = installation.Install(ctx, installWatcher()) testza.AssertNoError(t, err) installation.Vanilla = true - err = installation.Install(ctx, nil) + err = installation.Install(ctx, installWatcher()) testza.AssertNoError(t, err) } } diff --git a/cli/resolving_test.go b/cli/resolving_test.go index ad586a9..51439c7 100644 --- a/cli/resolving_test.go +++ b/cli/resolving_test.go @@ -6,8 +6,15 @@ import ( "testing" "github.com/MarvinJWendt/testza" + "github.com/rs/zerolog/log" + + "github.com/satisfactorymodding/ficsit-cli/cfg" ) +func init() { + cfg.SetDefaults() +} + func profilesGetResolver() DependencyResolver { ctx, err := InitCLI(false) if err != nil { @@ -17,6 +24,22 @@ func profilesGetResolver() DependencyResolver { return NewDependencyResolver(ctx.Provider) } +func installWatcher() chan<- InstallUpdate { + c := make(chan InstallUpdate) + go func() { + for i := range c { + if i.Progress.Total == i.Progress.Completed { + if i.Type != InstallUpdateTypeOverall { + log.Info().Str("mod_reference", i.Item.Mod).Str("version", i.Item.Version).Str("type", string(i.Type)).Msg("progress completed") + } else { + log.Info().Msg("overall completed") + } + } + } + }() + return c +} + func TestProfileResolution(t *testing.T) { resolver := profilesGetResolver() @@ -108,7 +131,7 @@ func TestUpdateMods(t *testing.T) { err = installation.WriteLockFile(ctx, oldLockfile) testza.AssertNoError(t, err) - err = installation.Install(ctx, nil) + err = installation.Install(ctx, installWatcher()) testza.AssertNoError(t, err) lockFile, err := installation.LockFile(ctx) @@ -126,7 +149,7 @@ func TestUpdateMods(t *testing.T) { testza.AssertEqual(t, 2, len(*lockFile)) testza.AssertEqual(t, "1.6.6", (*lockFile)["AreaActions"].Version) - err = installation.Install(ctx, nil) + err = installation.Install(ctx, installWatcher()) testza.AssertNoError(t, err) } } diff --git a/cmd/root.go b/cmd/root.go index e6c5f1f..9a1b52f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -139,6 +139,7 @@ func init() { RootCmd.PersistentFlags().String("api-key", "", "API key to use when sending requests") RootCmd.PersistentFlags().Bool("offline", false, "Whether to only use local data") + RootCmd.PersistentFlags().Int("concurrent-downloads", 5, "Maximum number of concurrent downloads") _ = viper.BindPFlag("log", RootCmd.PersistentFlags().Lookup("log")) _ = viper.BindPFlag("log-file", RootCmd.PersistentFlags().Lookup("log-file")) @@ -157,4 +158,5 @@ func init() { _ = viper.BindPFlag("api-key", RootCmd.PersistentFlags().Lookup("api-key")) _ = viper.BindPFlag("offline", RootCmd.PersistentFlags().Lookup("offline")) + _ = viper.BindPFlag("concurrent-downloads", RootCmd.PersistentFlags().Lookup("concurrent-downloads")) } diff --git a/go.mod b/go.mod index 338d767..91434fc 100644 --- a/go.mod +++ b/go.mod @@ -13,14 +13,16 @@ require ( github.com/charmbracelet/glamour v0.5.0 github.com/charmbracelet/lipgloss v0.6.0 github.com/jlaffaye/ftp v0.1.0 - github.com/mircearoata/pubgrub-go v0.3.2 + github.com/mircearoata/pubgrub-go v0.3.3 github.com/muesli/reflow v0.3.0 github.com/pkg/errors v0.9.1 github.com/pterm/pterm v0.12.67 + github.com/puzpuzpuz/xsync/v3 v3.0.2 github.com/rs/zerolog v1.28.0 github.com/sahilm/fuzzy v0.1.0 github.com/spf13/cobra v1.6.0 github.com/spf13/viper v1.13.0 + golang.org/x/sync v0.1.0 ) require ( diff --git a/go.sum b/go.sum index a2b5a64..29fddef 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/mircearoata/pubgrub-go v0.3.2 h1:AeC2bvHebii6YrfyN+AST06WiQUENKQkREA7Xoi4/HY= -github.com/mircearoata/pubgrub-go v0.3.2/go.mod h1:9oWL9ZXdjFYvnGl95qiM1dTciFNx1MN8fUnG3SUwDi8= +github.com/mircearoata/pubgrub-go v0.3.3 h1:XGwL8Xh5GX+mbnvWItbM/lVJxAq3NZtfUtbJ/hUf2ig= +github.com/mircearoata/pubgrub-go v0.3.3/go.mod h1:9oWL9ZXdjFYvnGl95qiM1dTciFNx1MN8fUnG3SUwDi8= github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -310,6 +310,8 @@ github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5b github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s= github.com/pterm/pterm v0.12.67 h1:5iB7ajIQROYfxYD7+sFJ4+KJhFJ+xn7QOVBm4s6RUF0= github.com/pterm/pterm v0.12.67/go.mod h1:nFuT9ZVkkCi8o4L1dtWuYPwDQxggLh4C263qG5nTLpQ= +github.com/puzpuzpuz/xsync/v3 v3.0.2 h1:3yESHrRFYr6xzkz61LLkvNiPFXxJEAABanTQpKbAaew= +github.com/puzpuzpuz/xsync/v3 v3.0.2/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= @@ -493,6 +495,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/tea/scenes/apply.go b/tea/scenes/apply.go index dd0a195..320dbf0 100644 --- a/tea/scenes/apply.go +++ b/tea/scenes/apply.go @@ -1,6 +1,8 @@ package scenes import ( + "sort" + "github.com/charmbracelet/bubbles/progress" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" @@ -9,120 +11,86 @@ import ( "github.com/satisfactorymodding/ficsit-cli/cli" "github.com/satisfactorymodding/ficsit-cli/tea/components" "github.com/satisfactorymodding/ficsit-cli/tea/scenes/keys" - "github.com/satisfactorymodding/ficsit-cli/tea/utils" + teaUtils "github.com/satisfactorymodding/ficsit-cli/tea/utils" + "github.com/satisfactorymodding/ficsit-cli/utils" ) var _ tea.Model = (*apply)(nil) -type update struct { - installName string - modName string - completed []string - installTotal int - installCurrent int - modTotal int - modCurrent int - done bool +type modProgress struct { + downloadProgress utils.GenericProgress + extractProgress utils.GenericProgress + downloading bool + complete bool +} + +type status struct { + modProgresses map[string]modProgress + installName string + overallProgress utils.GenericProgress + done bool } type apply struct { - root components.RootModel - parent tea.Model - error *components.ErrorComponent - updateChannel chan update - errorChannel chan error - cancelChannel chan bool - title string - status update - overall progress.Model - sub progress.Model - cancelled bool + root components.RootModel + parent tea.Model + error *components.ErrorComponent + installChannel chan string + updateChannel chan cli.InstallUpdate + doneChannel chan bool + errorChannel chan error + cancelChannel chan bool + title string + status status + overall progress.Model + sub progress.Model + cancelled bool } func NewApply(root components.RootModel, parent tea.Model) tea.Model { overall := progress.New(progress.WithSolidFill("118")) sub := progress.New(progress.WithSolidFill("202")) - updateChannel := make(chan update) + installChannel := make(chan string) + updateChannel := make(chan cli.InstallUpdate) + doneChannel := make(chan bool, 1) errorChannel := make(chan error) cancelChannel := make(chan bool, 1) model := &apply{ root: root, parent: parent, - title: utils.NonListTitleStyle.MarginTop(1).MarginBottom(1).Render("Applying Changes"), + title: teaUtils.NonListTitleStyle.MarginTop(1).MarginBottom(1).Render("Applying Changes"), overall: overall, sub: sub, - status: update{ - completed: []string{}, - - installName: "", - installTotal: 100, - installCurrent: 0, - - modName: "", - modTotal: 100, - modCurrent: 0, - - done: false, + status: status{ + installName: "", + done: false, }, - updateChannel: updateChannel, - errorChannel: errorChannel, - cancelChannel: cancelChannel, - cancelled: false, + installChannel: installChannel, + updateChannel: updateChannel, + doneChannel: doneChannel, + errorChannel: errorChannel, + cancelChannel: cancelChannel, + cancelled: false, } go func() { - result := &update{ - completed: make([]string, 0), - - installName: "", - installTotal: 100, - installCurrent: 0, - - modName: "", - modTotal: 100, - modCurrent: 0, - - done: false, - } - updateChannel <- *result - for _, installation := range root.GetGlobal().Installations.Installations { - result.installName = installation.Path - updateChannel <- *result - - installChannel := make(chan cli.InstallUpdate) + installChannel <- installation.Path + installUpdateChannel := make(chan cli.InstallUpdate) go func() { - for data := range installChannel { - result.installName = installation.Path - result.installCurrent = int(data.OverallProgress * 100) - - if data.DownloadProgress < 1 { - result.modName = "Downloading: " + data.ModName - result.modCurrent = int(data.DownloadProgress * 100) - } else { - result.modName = "Extracting: " + data.ModName - result.modCurrent = int(data.ExtractProgress * 100) - } - - updateChannel <- *result + for update := range installUpdateChannel { + updateChannel <- update } }() - if err := installation.Install(root.GetGlobal(), installChannel); err != nil { + if err := installation.Install(root.GetGlobal(), installUpdateChannel); err != nil { errorChannel <- err return } - close(installChannel) - - result.modName = "" - result.installTotal = 100 - result.completed = append(result.completed, installation.Path) - updateChannel <- *result - stop := false select { case <-cancelChannel: @@ -135,16 +103,14 @@ func NewApply(root components.RootModel, parent tea.Model) tea.Model { } } - result.done = true - result.installName = "" - updateChannel <- *result + doneChannel <- true }() return model } func (m apply) Init() tea.Cmd { - return utils.Ticker() + return teaUtils.Ticker() } func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) { @@ -169,10 +135,38 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.root.SetSize(msg) case components.ErrorComponentTimeoutMsg: m.error = nil - case utils.TickMsg: + case teaUtils.TickMsg: select { - case newStatus := <-m.updateChannel: - m.status = newStatus + case <-m.doneChannel: + m.status.done = true + m.status.installName = "" + break + case installName := <-m.installChannel: + m.status.installName = installName + m.status.modProgresses = make(map[string]modProgress) + m.status.overallProgress = utils.GenericProgress{} + break + case update := <-m.updateChannel: + switch update.Type { + case cli.InstallUpdateTypeOverall: + m.status.overallProgress = update.Progress + case cli.InstallUpdateTypeModDownload: + m.status.modProgresses[update.Item.Mod] = modProgress{ + downloadProgress: update.Progress, + downloading: true, + complete: false, + } + case cli.InstallUpdateTypeModExtract: + m.status.modProgresses[update.Item.Mod] = modProgress{ + extractProgress: update.Progress, + downloading: false, + complete: false, + } + case cli.InstallUpdateTypeModComplete: + m.status.modProgresses[update.Item.Mod] = modProgress{ + complete: true, + } + } break case err := <-m.errorChannel: wrappedErrMessage := wrap.String(err.Error(), int(float64(m.root.Size().Width)*0.8)) @@ -183,7 +177,7 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) { // Skip if nothing there break } - return m, utils.Ticker() + return m, teaUtils.Ticker() } return m, nil @@ -191,30 +185,38 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) { func (m apply) View() string { strs := make([]string, 0) - for _, s := range m.status.completed { - strs = append(strs, lipgloss.NewStyle().Foreground(lipgloss.Color("22")).Render("✓ ")+s) - } if m.status.installName != "" { - marginTop := 0 - if len(m.status.completed) > 0 { - marginTop = 1 - } - - strs = append(strs, lipgloss.NewStyle().MarginTop(marginTop).Render(m.status.installName)) - strs = append(strs, m.overall.ViewAs(float64(m.status.installCurrent)/float64(m.status.installTotal))) + strs = append(strs, lipgloss.NewStyle().Render(m.status.installName)) + strs = append(strs, lipgloss.NewStyle().MarginBottom(1).Render(m.overall.ViewAs(m.status.overallProgress.Percentage()))) } - if m.status.modName != "" { - strs = append(strs, lipgloss.NewStyle().MarginTop(1).Render(m.status.modName)) - strs = append(strs, m.sub.ViewAs(float64(m.status.modCurrent)/float64(m.status.modTotal))) + keys := make([]string, 0) + for k := range m.status.modProgresses { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, modReference := range keys { + p := m.status.modProgresses[modReference] + if p.complete { + strs = append(strs, lipgloss.NewStyle().Foreground(lipgloss.Color("22")).Render("✓ ")+modReference) + } else { + if p.downloading { + strs = append(strs, lipgloss.NewStyle().Render(modReference+" (Downloading)")) + strs = append(strs, m.sub.ViewAs(p.downloadProgress.Percentage())) + } else { + strs = append(strs, lipgloss.NewStyle().Render(modReference+" (Extracting)")) + strs = append(strs, m.sub.ViewAs(p.extractProgress.Percentage())) + } + } } if m.status.done { if m.cancelled { - strs = append(strs, utils.LabelStyle.Copy().Foreground(lipgloss.Color("196")).Padding(0).Margin(1).Render("Cancelled! Press Enter to return")) + strs = append(strs, teaUtils.LabelStyle.Copy().Foreground(lipgloss.Color("196")).Padding(0).Margin(1).Render("Cancelled! Press Enter to return")) } else { - strs = append(strs, utils.LabelStyle.Copy().Padding(0).Margin(1).Render("Done! Press Enter to return")) + strs = append(strs, teaUtils.LabelStyle.Copy().Padding(0).Margin(1).Render("Done! Press Enter to return")) } } diff --git a/utils/io.go b/utils/io.go index 7c07983..39c8eff 100644 --- a/utils/io.go +++ b/utils/io.go @@ -7,15 +7,51 @@ import ( "io" "os" "path/filepath" + "sync" + "sync/atomic" "github.com/pkg/errors" "github.com/satisfactorymodding/ficsit-cli/cli/disk" ) -type GenericUpdate struct { - ModReference *string - Progress float64 +type GenericProgress struct { + Completed int64 + Total int64 +} + +func (gp GenericProgress) Percentage() float64 { + if gp.Total == 0 { + return 0 + } + return float64(gp.Completed) / float64(gp.Total) +} + +type Progresser struct { + io.Reader + Updates chan<- GenericProgress + Total int64 + Running int64 +} + +func (pt *Progresser) Read(p []byte) (int, error) { + n, err := pt.Reader.Read(p) + pt.Running += int64(n) + + if err == nil { + if pt.Updates != nil { + select { + case pt.Updates <- GenericProgress{Completed: pt.Running, Total: pt.Total}: + default: + } + } + } + + if err == io.EOF { + return n, io.EOF + } + + return n, errors.Wrap(err, "failed to read") } func SHA256Data(f io.Reader) (string, error) { @@ -27,7 +63,7 @@ func SHA256Data(f io.Reader) (string, error) { return hex.EncodeToString(h.Sum(nil)), nil } -func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates chan GenericUpdate, d disk.Disk) error { +func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates chan<- GenericProgress, d disk.Disk) error { hashFile := filepath.Join(location, ".smm") hashBytes, err := d.Read(hashFile) if err != nil { @@ -59,7 +95,24 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates return errors.Wrap(err, "failed to read file as zip") } - for i, file := range reader.File { + totalSize := int64(0) + + for _, file := range reader.File { + totalSize += int64(file.UncompressedSize64) + } + + totalExtracted := int64(0) + totalExtractedPtr := &totalExtracted + + channelUsers := sync.WaitGroup{} + + if updates != nil { + defer func() { + channelUsers.Wait() + }() + } + + for _, file := range reader.File { if !file.FileInfo().IsDir() { outFileLocation := filepath.Join(location, file.Name) @@ -67,16 +120,26 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates return errors.Wrap(err, "failed to create mod directory: "+location) } - if err := writeZipFile(outFileLocation, file, d); err != nil { + var fileUpdates chan GenericProgress + if updates != nil { + fileUpdates = make(chan GenericProgress) + channelUsers.Add(1) + go func() { + defer channelUsers.Done() + for fileUpdate := range fileUpdates { + updates <- GenericProgress{ + Completed: atomic.LoadInt64(totalExtractedPtr) + fileUpdate.Completed, + Total: totalSize, + } + } + }() + } + + if err := writeZipFile(outFileLocation, file, d, fileUpdates); err != nil { return err } - } - if updates != nil { - select { - case updates <- GenericUpdate{Progress: float64(i) / float64(len(reader.File)-1)}: - default: - } + atomic.AddInt64(totalExtractedPtr, int64(file.UncompressedSize64)) } } @@ -86,7 +149,7 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates if updates != nil { select { - case updates <- GenericUpdate{Progress: 1}: + case updates <- GenericProgress{Completed: totalSize, Total: totalSize}: default: } } @@ -94,7 +157,11 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates return nil } -func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk) error { +func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk, updates chan<- GenericProgress) error { + if updates != nil { + defer close(updates) + } + outFile, err := d.Open(outFileLocation, os.O_CREATE|os.O_RDWR) if err != nil { return errors.Wrap(err, "failed to write to file: "+outFileLocation) @@ -106,8 +173,15 @@ func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk) error { if err != nil { return errors.Wrap(err, "failed to process mod zip") } + defer inFile.Close() - if _, err := io.Copy(outFile, inFile); err != nil { + progressInReader := &Progresser{ + Reader: inFile, + Total: int64(file.UncompressedSize64), + Updates: updates, + } + + if _, err := io.Copy(outFile, progressInReader); err != nil { return errors.Wrap(err, "failed to write to file: "+outFileLocation) }