feat: parallel downloads (#43)
* feat: parallel downloads * feat: mod extract progress using file size * feat: pass mod version in install progress updates * fix: only close update channels after finished sending * chore: verbose ci tests * fix: store mod in cache chore: add progress logging to tests * chore: lint * test: add concurrent download limit * fix: prevent concurrent map access * chore: bump pubgrub fix: fix race conditions --------- Co-authored-by: Vilsol <me@vil.so>
This commit is contained in:
mircearoata
GitHub
parent
a192a63c82
commit
6088d1e8eb
12 changed files with 401 additions and 222 deletions
2
.github/workflows/push.yaml
vendored
2
.github/workflows/push.yaml
vendored
|
|
@ -88,6 +88,6 @@ jobs:
|
|||
run: go generate -tags tools -x ./...
|
||||
|
||||
- name: Test
|
||||
run: go test ./...
|
||||
run: go test -v ./...
|
||||
env:
|
||||
SF_DEDICATED_SERVER: ${{ github.workspace }}/SatisfactoryDedicatedServer
|
||||
|
|
|
|||
|
|
@ -17,4 +17,5 @@ func SetDefaults() {
|
|||
viper.SetDefault("dry-run", false)
|
||||
viper.SetDefault("api-base", "https://api.ficsit.app")
|
||||
viper.SetDefault("graphql-api", "/v2/query")
|
||||
viper.SetDefault("concurrent-downloads", 5)
|
||||
}
|
||||
|
|
|
|||
51
cli/cache/download.go
vendored
51
cli/cache/download.go
vendored
|
|
@ -13,34 +13,11 @@ import (
|
|||
"github.com/satisfactorymodding/ficsit-cli/utils"
|
||||
)
|
||||
|
||||
type Progresser struct {
|
||||
io.Reader
|
||||
updates chan utils.GenericUpdate
|
||||
total int64
|
||||
running int64
|
||||
}
|
||||
|
||||
func (pt *Progresser) Read(p []byte) (int, error) {
|
||||
n, err := pt.Reader.Read(p)
|
||||
pt.running += int64(n)
|
||||
|
||||
if err == nil {
|
||||
if pt.updates != nil {
|
||||
select {
|
||||
case pt.updates <- utils.GenericUpdate{Progress: float64(pt.running) / float64(pt.total)}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
func DownloadOrCache(cacheKey string, hash string, url string, updates chan<- utils.GenericProgress, downloadSemaphore chan int) (io.ReaderAt, int64, error) {
|
||||
if updates != nil {
|
||||
defer close(updates)
|
||||
}
|
||||
|
||||
if err == io.EOF {
|
||||
return n, io.EOF
|
||||
}
|
||||
|
||||
return n, errors.Wrap(err, "failed to read")
|
||||
}
|
||||
|
||||
func DownloadOrCache(cacheKey string, hash string, url string, updates chan utils.GenericUpdate) (io.ReaderAt, int64, error) {
|
||||
downloadCache := filepath.Join(viper.GetString("cache-dir"), "downloadCache")
|
||||
if err := os.MkdirAll(downloadCache, 0o777); err != nil {
|
||||
if !os.IsExist(err) {
|
||||
|
|
@ -82,6 +59,20 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
|
|||
return nil, 0, errors.Wrap(err, "failed to stat file: "+location)
|
||||
}
|
||||
|
||||
if updates != nil {
|
||||
headResp, err := http.Head(url)
|
||||
if err != nil {
|
||||
return nil, 0, errors.Wrap(err, "failed to head: "+url)
|
||||
}
|
||||
defer headResp.Body.Close()
|
||||
updates <- utils.GenericProgress{Total: headResp.ContentLength}
|
||||
}
|
||||
|
||||
if downloadSemaphore != nil {
|
||||
downloadSemaphore <- 1
|
||||
defer func() { <-downloadSemaphore }()
|
||||
}
|
||||
|
||||
out, err := os.Create(location)
|
||||
if err != nil {
|
||||
return nil, 0, errors.Wrap(err, "failed creating file at: "+location)
|
||||
|
|
@ -98,10 +89,10 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
|
|||
return nil, 0, fmt.Errorf("bad status: %s on url: %s", resp.Status, url)
|
||||
}
|
||||
|
||||
progresser := &Progresser{
|
||||
progresser := &utils.Progresser{
|
||||
Reader: resp.Body,
|
||||
total: resp.ContentLength,
|
||||
updates: updates,
|
||||
Total: resp.ContentLength,
|
||||
Updates: updates,
|
||||
}
|
||||
|
||||
_, err = io.Copy(out, progresser)
|
||||
|
|
@ -116,7 +107,7 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
|
|||
|
||||
if updates != nil {
|
||||
select {
|
||||
case updates <- utils.GenericUpdate{Progress: 1}:
|
||||
case updates <- utils.GenericProgress{Completed: resp.ContentLength, Total: resp.ContentLength}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"github.com/mircearoata/pubgrub-go/pubgrub/helpers"
|
||||
"github.com/mircearoata/pubgrub-go/pubgrub/semver"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/satisfactorymodding/ficsit-cli/cli/provider"
|
||||
|
|
@ -35,7 +36,7 @@ type ficsitAPISource struct {
|
|||
provider provider.Provider
|
||||
lockfile *LockFile
|
||||
toInstall map[string]semver.Constraint
|
||||
modVersionInfo map[string]ficsit.ModVersionsWithDependenciesResponse
|
||||
modVersionInfo *xsync.MapOf[string, ficsit.ModVersionsWithDependenciesResponse]
|
||||
gameVersion semver.Version
|
||||
smlVersions []ficsit.SMLVersionsSmlVersionsGetSMLVersionsSml_versionsSMLVersion
|
||||
}
|
||||
|
|
@ -74,7 +75,7 @@ func (f *ficsitAPISource) GetPackageVersions(pkg string) ([]pubgrub.PackageVersi
|
|||
if response.Mod.Id == "" {
|
||||
return nil, errors.Errorf("mod %s not found", pkg)
|
||||
}
|
||||
f.modVersionInfo[pkg] = *response
|
||||
f.modVersionInfo.Store(pkg, *response)
|
||||
versions := make([]pubgrub.PackageVersion, len(response.Mod.Versions))
|
||||
for i, modVersion := range response.Mod.Versions {
|
||||
v, err := semver.NewVersion(modVersion.Version)
|
||||
|
|
@ -145,7 +146,7 @@ func (d DependencyResolver) ResolveModDependencies(constraints map[string]string
|
|||
gameVersion: gameVersionSemver,
|
||||
lockfile: lockFile,
|
||||
toInstall: toInstall,
|
||||
modVersionInfo: make(map[string]ficsit.ModVersionsWithDependenciesResponse),
|
||||
modVersionInfo: xsync.NewMapOf[string, ficsit.ModVersionsWithDependenciesResponse](),
|
||||
}
|
||||
|
||||
result, err := pubgrub.Solve(helpers.NewCachingSource(ficsitSource), rootPkg)
|
||||
|
|
@ -170,7 +171,8 @@ func (d DependencyResolver) ResolveModDependencies(constraints map[string]string
|
|||
}
|
||||
continue
|
||||
}
|
||||
versions := ficsitSource.modVersionInfo[k].Mod.Versions
|
||||
value, _ := ficsitSource.modVersionInfo.Load(k)
|
||||
versions := value.Mod.Versions
|
||||
for _, ver := range versions {
|
||||
if ver.Version == v.RawString() {
|
||||
outputLock[k] = LockedMod{
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import (
|
|||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/spf13/viper"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/satisfactorymodding/ficsit-cli/cli/cache"
|
||||
"github.com/satisfactorymodding/ficsit-cli/cli/disk"
|
||||
|
|
@ -351,14 +352,27 @@ func (i *Installation) ResolveProfile(ctx *GlobalContext) (LockFile, error) {
|
|||
return lockfile, nil
|
||||
}
|
||||
|
||||
type InstallUpdateType string
|
||||
|
||||
var (
|
||||
InstallUpdateTypeOverall InstallUpdateType = "overall"
|
||||
InstallUpdateTypeModDownload InstallUpdateType = "download"
|
||||
InstallUpdateTypeModExtract InstallUpdateType = "extract"
|
||||
InstallUpdateTypeModComplete InstallUpdateType = "complete"
|
||||
)
|
||||
|
||||
type InstallUpdate struct {
|
||||
ModName string
|
||||
OverallProgress float64
|
||||
DownloadProgress float64
|
||||
ExtractProgress float64
|
||||
Type InstallUpdateType
|
||||
Item InstallUpdateItem
|
||||
Progress utils.GenericProgress
|
||||
}
|
||||
|
||||
func (i *Installation) Install(ctx *GlobalContext, updates chan InstallUpdate) error {
|
||||
type InstallUpdateItem struct {
|
||||
Mod string
|
||||
Version string
|
||||
}
|
||||
|
||||
func (i *Installation) Install(ctx *GlobalContext, updates chan<- InstallUpdate) error {
|
||||
if err := i.Validate(ctx); err != nil {
|
||||
return errors.Wrap(err, "failed to validate installation")
|
||||
}
|
||||
|
|
@ -403,78 +417,65 @@ func (i *Installation) Install(ctx *GlobalContext, updates chan InstallUpdate) e
|
|||
}
|
||||
}
|
||||
|
||||
downloading := true
|
||||
completed := 0
|
||||
log.Info().Int("concurrency", viper.GetInt("concurrent-downloads")).Str("path", i.Path).Msg("starting installation")
|
||||
|
||||
var genericUpdates chan utils.GenericUpdate
|
||||
errg := errgroup.Group{}
|
||||
channelUsers := sync.WaitGroup{}
|
||||
downloadSemaphore := make(chan int, viper.GetInt("concurrent-downloads"))
|
||||
defer close(downloadSemaphore)
|
||||
|
||||
var modComplete chan int
|
||||
if updates != nil {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
defer wg.Wait()
|
||||
|
||||
genericUpdates = make(chan utils.GenericUpdate)
|
||||
defer close(genericUpdates)
|
||||
|
||||
channelUsers.Add(1)
|
||||
modComplete = make(chan int)
|
||||
defer close(modComplete)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
update := InstallUpdate{
|
||||
OverallProgress: float64(completed) / float64(len(lockfile)),
|
||||
DownloadProgress: 0,
|
||||
ExtractProgress: 0,
|
||||
}
|
||||
|
||||
select {
|
||||
case updates <- update:
|
||||
default:
|
||||
}
|
||||
|
||||
for up := range genericUpdates {
|
||||
if downloading {
|
||||
update.DownloadProgress = up.Progress
|
||||
} else {
|
||||
update.DownloadProgress = 1
|
||||
update.ExtractProgress = up.Progress
|
||||
}
|
||||
|
||||
if up.ModReference != nil {
|
||||
update.ModName = *up.ModReference
|
||||
}
|
||||
|
||||
update.OverallProgress = float64(completed) / float64(len(lockfile))
|
||||
|
||||
select {
|
||||
case updates <- update:
|
||||
default:
|
||||
defer channelUsers.Done()
|
||||
completed := 0
|
||||
for range modComplete {
|
||||
completed++
|
||||
overallUpdate := InstallUpdate{
|
||||
Type: InstallUpdateTypeOverall,
|
||||
Progress: utils.GenericProgress{
|
||||
Completed: int64(completed),
|
||||
Total: int64(len(lockfile)),
|
||||
},
|
||||
}
|
||||
updates <- overallUpdate
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for modReference, version := range lockfile {
|
||||
channelUsers.Add(1)
|
||||
modReference := modReference
|
||||
version := version
|
||||
errg.Go(func() error {
|
||||
defer channelUsers.Done()
|
||||
// Only install if a link is provided, otherwise assume mod is already installed
|
||||
if version.Link != "" {
|
||||
downloading = true
|
||||
|
||||
if genericUpdates != nil {
|
||||
genericUpdates <- utils.GenericUpdate{ModReference: &modReference}
|
||||
}
|
||||
|
||||
log.Info().Str("mod_reference", modReference).Str("version", version.Version).Str("link", version.Link).Msg("downloading mod")
|
||||
reader, size, err := cache.DownloadOrCache(modReference+"_"+version.Version+".zip", version.Hash, version.Link, genericUpdates)
|
||||
err := downloadAndExtractMod(modReference, version.Version, version.Link, version.Hash, modsDirectory, updates, downloadSemaphore, d)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to download "+modReference+" from: "+version.Link)
|
||||
}
|
||||
|
||||
downloading = false
|
||||
|
||||
log.Info().Str("mod_reference", modReference).Str("version", version.Version).Str("link", version.Link).Msg("extracting mod")
|
||||
if err := utils.ExtractMod(reader, size, filepath.Join(modsDirectory, modReference), version.Hash, genericUpdates, d); err != nil {
|
||||
return errors.Wrap(err, "could not extract "+modReference)
|
||||
return errors.Wrapf(err, "failed to install %s@%s", modReference, version.Version)
|
||||
}
|
||||
}
|
||||
|
||||
completed++
|
||||
if modComplete != nil {
|
||||
modComplete <- 1
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if updates != nil {
|
||||
go func() {
|
||||
channelUsers.Wait()
|
||||
close(updates)
|
||||
}()
|
||||
}
|
||||
|
||||
if err := errg.Wait(); err != nil {
|
||||
return errors.Wrap(err, "failed to install mods")
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -518,6 +519,84 @@ func (i *Installation) UpdateMods(ctx *GlobalContext, mods []string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func downloadAndExtractMod(modReference string, version string, link string, hash string, modsDirectory string, updates chan<- InstallUpdate, downloadSemaphore chan int, d disk.Disk) error {
|
||||
var downloadUpdates chan utils.GenericProgress
|
||||
|
||||
if updates != nil {
|
||||
// Forward the inner updates as InstallUpdates
|
||||
downloadUpdates = make(chan utils.GenericProgress)
|
||||
|
||||
go func() {
|
||||
for up := range downloadUpdates {
|
||||
updates <- InstallUpdate{
|
||||
Item: InstallUpdateItem{
|
||||
Mod: modReference,
|
||||
Version: version,
|
||||
},
|
||||
Type: InstallUpdateTypeModDownload,
|
||||
Progress: up,
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
log.Info().Str("mod_reference", modReference).Str("version", version).Str("link", link).Msg("downloading mod")
|
||||
reader, size, err := cache.DownloadOrCache(modReference+"_"+version+".zip", hash, link, downloadUpdates, downloadSemaphore)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to download "+modReference+" from: "+link)
|
||||
}
|
||||
|
||||
var extractUpdates chan utils.GenericProgress
|
||||
|
||||
var wg sync.WaitGroup
|
||||
if updates != nil {
|
||||
// Forward the inner updates as InstallUpdates
|
||||
extractUpdates = make(chan utils.GenericProgress)
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for up := range extractUpdates {
|
||||
select {
|
||||
case updates <- InstallUpdate{
|
||||
Item: InstallUpdateItem{
|
||||
Mod: modReference,
|
||||
Version: version,
|
||||
},
|
||||
Type: InstallUpdateTypeModExtract,
|
||||
Progress: up,
|
||||
}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
log.Info().Str("mod_reference", modReference).Str("version", version).Str("link", link).Msg("extracting mod")
|
||||
if err := utils.ExtractMod(reader, size, filepath.Join(modsDirectory, modReference), hash, extractUpdates, d); err != nil {
|
||||
return errors.Wrap(err, "could not extract "+modReference)
|
||||
}
|
||||
|
||||
if updates != nil {
|
||||
select {
|
||||
case updates <- InstallUpdate{
|
||||
Type: InstallUpdateTypeModComplete,
|
||||
Item: InstallUpdateItem{
|
||||
Mod: modReference,
|
||||
Version: version,
|
||||
},
|
||||
}:
|
||||
default:
|
||||
}
|
||||
|
||||
close(extractUpdates)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (i *Installation) SetProfile(ctx *GlobalContext, profile string) error {
|
||||
found := false
|
||||
for _, p := range ctx.Profiles.Profiles {
|
||||
|
|
|
|||
|
|
@ -35,11 +35,11 @@ func TestAddInstallation(t *testing.T) {
|
|||
testza.AssertNoError(t, err)
|
||||
testza.AssertNotNil(t, installation)
|
||||
|
||||
err = installation.Install(ctx, nil)
|
||||
err = installation.Install(ctx, installWatcher())
|
||||
testza.AssertNoError(t, err)
|
||||
|
||||
installation.Vanilla = true
|
||||
err = installation.Install(ctx, nil)
|
||||
err = installation.Install(ctx, installWatcher())
|
||||
testza.AssertNoError(t, err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,15 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/MarvinJWendt/testza"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"github.com/satisfactorymodding/ficsit-cli/cfg"
|
||||
)
|
||||
|
||||
func init() {
|
||||
cfg.SetDefaults()
|
||||
}
|
||||
|
||||
func profilesGetResolver() DependencyResolver {
|
||||
ctx, err := InitCLI(false)
|
||||
if err != nil {
|
||||
|
|
@ -17,6 +24,22 @@ func profilesGetResolver() DependencyResolver {
|
|||
return NewDependencyResolver(ctx.Provider)
|
||||
}
|
||||
|
||||
func installWatcher() chan<- InstallUpdate {
|
||||
c := make(chan InstallUpdate)
|
||||
go func() {
|
||||
for i := range c {
|
||||
if i.Progress.Total == i.Progress.Completed {
|
||||
if i.Type != InstallUpdateTypeOverall {
|
||||
log.Info().Str("mod_reference", i.Item.Mod).Str("version", i.Item.Version).Str("type", string(i.Type)).Msg("progress completed")
|
||||
} else {
|
||||
log.Info().Msg("overall completed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return c
|
||||
}
|
||||
|
||||
func TestProfileResolution(t *testing.T) {
|
||||
resolver := profilesGetResolver()
|
||||
|
||||
|
|
@ -108,7 +131,7 @@ func TestUpdateMods(t *testing.T) {
|
|||
err = installation.WriteLockFile(ctx, oldLockfile)
|
||||
testza.AssertNoError(t, err)
|
||||
|
||||
err = installation.Install(ctx, nil)
|
||||
err = installation.Install(ctx, installWatcher())
|
||||
testza.AssertNoError(t, err)
|
||||
|
||||
lockFile, err := installation.LockFile(ctx)
|
||||
|
|
@ -126,7 +149,7 @@ func TestUpdateMods(t *testing.T) {
|
|||
testza.AssertEqual(t, 2, len(*lockFile))
|
||||
testza.AssertEqual(t, "1.6.6", (*lockFile)["AreaActions"].Version)
|
||||
|
||||
err = installation.Install(ctx, nil)
|
||||
err = installation.Install(ctx, installWatcher())
|
||||
testza.AssertNoError(t, err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -139,6 +139,7 @@ func init() {
|
|||
RootCmd.PersistentFlags().String("api-key", "", "API key to use when sending requests")
|
||||
|
||||
RootCmd.PersistentFlags().Bool("offline", false, "Whether to only use local data")
|
||||
RootCmd.PersistentFlags().Int("concurrent-downloads", 5, "Maximum number of concurrent downloads")
|
||||
|
||||
_ = viper.BindPFlag("log", RootCmd.PersistentFlags().Lookup("log"))
|
||||
_ = viper.BindPFlag("log-file", RootCmd.PersistentFlags().Lookup("log-file"))
|
||||
|
|
@ -157,4 +158,5 @@ func init() {
|
|||
_ = viper.BindPFlag("api-key", RootCmd.PersistentFlags().Lookup("api-key"))
|
||||
|
||||
_ = viper.BindPFlag("offline", RootCmd.PersistentFlags().Lookup("offline"))
|
||||
_ = viper.BindPFlag("concurrent-downloads", RootCmd.PersistentFlags().Lookup("concurrent-downloads"))
|
||||
}
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -13,14 +13,16 @@ require (
|
|||
github.com/charmbracelet/glamour v0.5.0
|
||||
github.com/charmbracelet/lipgloss v0.6.0
|
||||
github.com/jlaffaye/ftp v0.1.0
|
||||
github.com/mircearoata/pubgrub-go v0.3.2
|
||||
github.com/mircearoata/pubgrub-go v0.3.3
|
||||
github.com/muesli/reflow v0.3.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/pterm/pterm v0.12.67
|
||||
github.com/puzpuzpuz/xsync/v3 v3.0.2
|
||||
github.com/rs/zerolog v1.28.0
|
||||
github.com/sahilm/fuzzy v0.1.0
|
||||
github.com/spf13/cobra v1.6.0
|
||||
github.com/spf13/viper v1.13.0
|
||||
golang.org/x/sync v0.1.0
|
||||
)
|
||||
|
||||
require (
|
||||
|
|
|
|||
7
go.sum
7
go.sum
|
|
@ -269,8 +269,8 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
|
|||
github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
|
||||
github.com/mircearoata/pubgrub-go v0.3.2 h1:AeC2bvHebii6YrfyN+AST06WiQUENKQkREA7Xoi4/HY=
|
||||
github.com/mircearoata/pubgrub-go v0.3.2/go.mod h1:9oWL9ZXdjFYvnGl95qiM1dTciFNx1MN8fUnG3SUwDi8=
|
||||
github.com/mircearoata/pubgrub-go v0.3.3 h1:XGwL8Xh5GX+mbnvWItbM/lVJxAq3NZtfUtbJ/hUf2ig=
|
||||
github.com/mircearoata/pubgrub-go v0.3.3/go.mod h1:9oWL9ZXdjFYvnGl95qiM1dTciFNx1MN8fUnG3SUwDi8=
|
||||
github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
|
|
@ -310,6 +310,8 @@ github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5b
|
|||
github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s=
|
||||
github.com/pterm/pterm v0.12.67 h1:5iB7ajIQROYfxYD7+sFJ4+KJhFJ+xn7QOVBm4s6RUF0=
|
||||
github.com/pterm/pterm v0.12.67/go.mod h1:nFuT9ZVkkCi8o4L1dtWuYPwDQxggLh4C263qG5nTLpQ=
|
||||
github.com/puzpuzpuz/xsync/v3 v3.0.2 h1:3yESHrRFYr6xzkz61LLkvNiPFXxJEAABanTQpKbAaew=
|
||||
github.com/puzpuzpuz/xsync/v3 v3.0.2/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
|
||||
|
|
@ -493,6 +495,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
package scenes
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/charmbracelet/bubbles/progress"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
|
|
@ -9,19 +11,23 @@ import (
|
|||
"github.com/satisfactorymodding/ficsit-cli/cli"
|
||||
"github.com/satisfactorymodding/ficsit-cli/tea/components"
|
||||
"github.com/satisfactorymodding/ficsit-cli/tea/scenes/keys"
|
||||
"github.com/satisfactorymodding/ficsit-cli/tea/utils"
|
||||
teaUtils "github.com/satisfactorymodding/ficsit-cli/tea/utils"
|
||||
"github.com/satisfactorymodding/ficsit-cli/utils"
|
||||
)
|
||||
|
||||
var _ tea.Model = (*apply)(nil)
|
||||
|
||||
type update struct {
|
||||
type modProgress struct {
|
||||
downloadProgress utils.GenericProgress
|
||||
extractProgress utils.GenericProgress
|
||||
downloading bool
|
||||
complete bool
|
||||
}
|
||||
|
||||
type status struct {
|
||||
modProgresses map[string]modProgress
|
||||
installName string
|
||||
modName string
|
||||
completed []string
|
||||
installTotal int
|
||||
installCurrent int
|
||||
modTotal int
|
||||
modCurrent int
|
||||
overallProgress utils.GenericProgress
|
||||
done bool
|
||||
}
|
||||
|
||||
|
|
@ -29,11 +35,13 @@ type apply struct {
|
|||
root components.RootModel
|
||||
parent tea.Model
|
||||
error *components.ErrorComponent
|
||||
updateChannel chan update
|
||||
installChannel chan string
|
||||
updateChannel chan cli.InstallUpdate
|
||||
doneChannel chan bool
|
||||
errorChannel chan error
|
||||
cancelChannel chan bool
|
||||
title string
|
||||
status update
|
||||
status status
|
||||
overall progress.Model
|
||||
sub progress.Model
|
||||
cancelled bool
|
||||
|
|
@ -43,86 +51,46 @@ func NewApply(root components.RootModel, parent tea.Model) tea.Model {
|
|||
overall := progress.New(progress.WithSolidFill("118"))
|
||||
sub := progress.New(progress.WithSolidFill("202"))
|
||||
|
||||
updateChannel := make(chan update)
|
||||
installChannel := make(chan string)
|
||||
updateChannel := make(chan cli.InstallUpdate)
|
||||
doneChannel := make(chan bool, 1)
|
||||
errorChannel := make(chan error)
|
||||
cancelChannel := make(chan bool, 1)
|
||||
|
||||
model := &apply{
|
||||
root: root,
|
||||
parent: parent,
|
||||
title: utils.NonListTitleStyle.MarginTop(1).MarginBottom(1).Render("Applying Changes"),
|
||||
title: teaUtils.NonListTitleStyle.MarginTop(1).MarginBottom(1).Render("Applying Changes"),
|
||||
overall: overall,
|
||||
sub: sub,
|
||||
status: update{
|
||||
completed: []string{},
|
||||
|
||||
status: status{
|
||||
installName: "",
|
||||
installTotal: 100,
|
||||
installCurrent: 0,
|
||||
|
||||
modName: "",
|
||||
modTotal: 100,
|
||||
modCurrent: 0,
|
||||
|
||||
done: false,
|
||||
},
|
||||
installChannel: installChannel,
|
||||
updateChannel: updateChannel,
|
||||
doneChannel: doneChannel,
|
||||
errorChannel: errorChannel,
|
||||
cancelChannel: cancelChannel,
|
||||
cancelled: false,
|
||||
}
|
||||
|
||||
go func() {
|
||||
result := &update{
|
||||
completed: make([]string, 0),
|
||||
|
||||
installName: "",
|
||||
installTotal: 100,
|
||||
installCurrent: 0,
|
||||
|
||||
modName: "",
|
||||
modTotal: 100,
|
||||
modCurrent: 0,
|
||||
|
||||
done: false,
|
||||
}
|
||||
updateChannel <- *result
|
||||
|
||||
for _, installation := range root.GetGlobal().Installations.Installations {
|
||||
result.installName = installation.Path
|
||||
updateChannel <- *result
|
||||
|
||||
installChannel := make(chan cli.InstallUpdate)
|
||||
installChannel <- installation.Path
|
||||
|
||||
installUpdateChannel := make(chan cli.InstallUpdate)
|
||||
go func() {
|
||||
for data := range installChannel {
|
||||
result.installName = installation.Path
|
||||
result.installCurrent = int(data.OverallProgress * 100)
|
||||
|
||||
if data.DownloadProgress < 1 {
|
||||
result.modName = "Downloading: " + data.ModName
|
||||
result.modCurrent = int(data.DownloadProgress * 100)
|
||||
} else {
|
||||
result.modName = "Extracting: " + data.ModName
|
||||
result.modCurrent = int(data.ExtractProgress * 100)
|
||||
}
|
||||
|
||||
updateChannel <- *result
|
||||
for update := range installUpdateChannel {
|
||||
updateChannel <- update
|
||||
}
|
||||
}()
|
||||
|
||||
if err := installation.Install(root.GetGlobal(), installChannel); err != nil {
|
||||
if err := installation.Install(root.GetGlobal(), installUpdateChannel); err != nil {
|
||||
errorChannel <- err
|
||||
return
|
||||
}
|
||||
|
||||
close(installChannel)
|
||||
|
||||
result.modName = ""
|
||||
result.installTotal = 100
|
||||
result.completed = append(result.completed, installation.Path)
|
||||
updateChannel <- *result
|
||||
|
||||
stop := false
|
||||
select {
|
||||
case <-cancelChannel:
|
||||
|
|
@ -135,16 +103,14 @@ func NewApply(root components.RootModel, parent tea.Model) tea.Model {
|
|||
}
|
||||
}
|
||||
|
||||
result.done = true
|
||||
result.installName = ""
|
||||
updateChannel <- *result
|
||||
doneChannel <- true
|
||||
}()
|
||||
|
||||
return model
|
||||
}
|
||||
|
||||
func (m apply) Init() tea.Cmd {
|
||||
return utils.Ticker()
|
||||
return teaUtils.Ticker()
|
||||
}
|
||||
|
||||
func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
|
|
@ -169,10 +135,38 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||
m.root.SetSize(msg)
|
||||
case components.ErrorComponentTimeoutMsg:
|
||||
m.error = nil
|
||||
case utils.TickMsg:
|
||||
case teaUtils.TickMsg:
|
||||
select {
|
||||
case newStatus := <-m.updateChannel:
|
||||
m.status = newStatus
|
||||
case <-m.doneChannel:
|
||||
m.status.done = true
|
||||
m.status.installName = ""
|
||||
break
|
||||
case installName := <-m.installChannel:
|
||||
m.status.installName = installName
|
||||
m.status.modProgresses = make(map[string]modProgress)
|
||||
m.status.overallProgress = utils.GenericProgress{}
|
||||
break
|
||||
case update := <-m.updateChannel:
|
||||
switch update.Type {
|
||||
case cli.InstallUpdateTypeOverall:
|
||||
m.status.overallProgress = update.Progress
|
||||
case cli.InstallUpdateTypeModDownload:
|
||||
m.status.modProgresses[update.Item.Mod] = modProgress{
|
||||
downloadProgress: update.Progress,
|
||||
downloading: true,
|
||||
complete: false,
|
||||
}
|
||||
case cli.InstallUpdateTypeModExtract:
|
||||
m.status.modProgresses[update.Item.Mod] = modProgress{
|
||||
extractProgress: update.Progress,
|
||||
downloading: false,
|
||||
complete: false,
|
||||
}
|
||||
case cli.InstallUpdateTypeModComplete:
|
||||
m.status.modProgresses[update.Item.Mod] = modProgress{
|
||||
complete: true,
|
||||
}
|
||||
}
|
||||
break
|
||||
case err := <-m.errorChannel:
|
||||
wrappedErrMessage := wrap.String(err.Error(), int(float64(m.root.Size().Width)*0.8))
|
||||
|
|
@ -183,7 +177,7 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||
// Skip if nothing there
|
||||
break
|
||||
}
|
||||
return m, utils.Ticker()
|
||||
return m, teaUtils.Ticker()
|
||||
}
|
||||
|
||||
return m, nil
|
||||
|
|
@ -191,30 +185,38 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||
|
||||
func (m apply) View() string {
|
||||
strs := make([]string, 0)
|
||||
for _, s := range m.status.completed {
|
||||
strs = append(strs, lipgloss.NewStyle().Foreground(lipgloss.Color("22")).Render("✓ ")+s)
|
||||
}
|
||||
|
||||
if m.status.installName != "" {
|
||||
marginTop := 0
|
||||
if len(m.status.completed) > 0 {
|
||||
marginTop = 1
|
||||
strs = append(strs, lipgloss.NewStyle().Render(m.status.installName))
|
||||
strs = append(strs, lipgloss.NewStyle().MarginBottom(1).Render(m.overall.ViewAs(m.status.overallProgress.Percentage())))
|
||||
}
|
||||
|
||||
strs = append(strs, lipgloss.NewStyle().MarginTop(marginTop).Render(m.status.installName))
|
||||
strs = append(strs, m.overall.ViewAs(float64(m.status.installCurrent)/float64(m.status.installTotal)))
|
||||
keys := make([]string, 0)
|
||||
for k := range m.status.modProgresses {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
if m.status.modName != "" {
|
||||
strs = append(strs, lipgloss.NewStyle().MarginTop(1).Render(m.status.modName))
|
||||
strs = append(strs, m.sub.ViewAs(float64(m.status.modCurrent)/float64(m.status.modTotal)))
|
||||
for _, modReference := range keys {
|
||||
p := m.status.modProgresses[modReference]
|
||||
if p.complete {
|
||||
strs = append(strs, lipgloss.NewStyle().Foreground(lipgloss.Color("22")).Render("✓ ")+modReference)
|
||||
} else {
|
||||
if p.downloading {
|
||||
strs = append(strs, lipgloss.NewStyle().Render(modReference+" (Downloading)"))
|
||||
strs = append(strs, m.sub.ViewAs(p.downloadProgress.Percentage()))
|
||||
} else {
|
||||
strs = append(strs, lipgloss.NewStyle().Render(modReference+" (Extracting)"))
|
||||
strs = append(strs, m.sub.ViewAs(p.extractProgress.Percentage()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if m.status.done {
|
||||
if m.cancelled {
|
||||
strs = append(strs, utils.LabelStyle.Copy().Foreground(lipgloss.Color("196")).Padding(0).Margin(1).Render("Cancelled! Press Enter to return"))
|
||||
strs = append(strs, teaUtils.LabelStyle.Copy().Foreground(lipgloss.Color("196")).Padding(0).Margin(1).Render("Cancelled! Press Enter to return"))
|
||||
} else {
|
||||
strs = append(strs, utils.LabelStyle.Copy().Padding(0).Margin(1).Render("Done! Press Enter to return"))
|
||||
strs = append(strs, teaUtils.LabelStyle.Copy().Padding(0).Margin(1).Render("Done! Press Enter to return"))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
102
utils/io.go
102
utils/io.go
|
|
@ -7,15 +7,51 @@ import (
|
|||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/satisfactorymodding/ficsit-cli/cli/disk"
|
||||
)
|
||||
|
||||
type GenericUpdate struct {
|
||||
ModReference *string
|
||||
Progress float64
|
||||
type GenericProgress struct {
|
||||
Completed int64
|
||||
Total int64
|
||||
}
|
||||
|
||||
func (gp GenericProgress) Percentage() float64 {
|
||||
if gp.Total == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(gp.Completed) / float64(gp.Total)
|
||||
}
|
||||
|
||||
type Progresser struct {
|
||||
io.Reader
|
||||
Updates chan<- GenericProgress
|
||||
Total int64
|
||||
Running int64
|
||||
}
|
||||
|
||||
func (pt *Progresser) Read(p []byte) (int, error) {
|
||||
n, err := pt.Reader.Read(p)
|
||||
pt.Running += int64(n)
|
||||
|
||||
if err == nil {
|
||||
if pt.Updates != nil {
|
||||
select {
|
||||
case pt.Updates <- GenericProgress{Completed: pt.Running, Total: pt.Total}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err == io.EOF {
|
||||
return n, io.EOF
|
||||
}
|
||||
|
||||
return n, errors.Wrap(err, "failed to read")
|
||||
}
|
||||
|
||||
func SHA256Data(f io.Reader) (string, error) {
|
||||
|
|
@ -27,7 +63,7 @@ func SHA256Data(f io.Reader) (string, error) {
|
|||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates chan GenericUpdate, d disk.Disk) error {
|
||||
func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates chan<- GenericProgress, d disk.Disk) error {
|
||||
hashFile := filepath.Join(location, ".smm")
|
||||
hashBytes, err := d.Read(hashFile)
|
||||
if err != nil {
|
||||
|
|
@ -59,7 +95,24 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
|
|||
return errors.Wrap(err, "failed to read file as zip")
|
||||
}
|
||||
|
||||
for i, file := range reader.File {
|
||||
totalSize := int64(0)
|
||||
|
||||
for _, file := range reader.File {
|
||||
totalSize += int64(file.UncompressedSize64)
|
||||
}
|
||||
|
||||
totalExtracted := int64(0)
|
||||
totalExtractedPtr := &totalExtracted
|
||||
|
||||
channelUsers := sync.WaitGroup{}
|
||||
|
||||
if updates != nil {
|
||||
defer func() {
|
||||
channelUsers.Wait()
|
||||
}()
|
||||
}
|
||||
|
||||
for _, file := range reader.File {
|
||||
if !file.FileInfo().IsDir() {
|
||||
outFileLocation := filepath.Join(location, file.Name)
|
||||
|
||||
|
|
@ -67,16 +120,26 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
|
|||
return errors.Wrap(err, "failed to create mod directory: "+location)
|
||||
}
|
||||
|
||||
if err := writeZipFile(outFileLocation, file, d); err != nil {
|
||||
return err
|
||||
var fileUpdates chan GenericProgress
|
||||
if updates != nil {
|
||||
fileUpdates = make(chan GenericProgress)
|
||||
channelUsers.Add(1)
|
||||
go func() {
|
||||
defer channelUsers.Done()
|
||||
for fileUpdate := range fileUpdates {
|
||||
updates <- GenericProgress{
|
||||
Completed: atomic.LoadInt64(totalExtractedPtr) + fileUpdate.Completed,
|
||||
Total: totalSize,
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if updates != nil {
|
||||
select {
|
||||
case updates <- GenericUpdate{Progress: float64(i) / float64(len(reader.File)-1)}:
|
||||
default:
|
||||
if err := writeZipFile(outFileLocation, file, d, fileUpdates); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
atomic.AddInt64(totalExtractedPtr, int64(file.UncompressedSize64))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -86,7 +149,7 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
|
|||
|
||||
if updates != nil {
|
||||
select {
|
||||
case updates <- GenericUpdate{Progress: 1}:
|
||||
case updates <- GenericProgress{Completed: totalSize, Total: totalSize}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
|
@ -94,7 +157,11 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
|
|||
return nil
|
||||
}
|
||||
|
||||
func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk) error {
|
||||
func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk, updates chan<- GenericProgress) error {
|
||||
if updates != nil {
|
||||
defer close(updates)
|
||||
}
|
||||
|
||||
outFile, err := d.Open(outFileLocation, os.O_CREATE|os.O_RDWR)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to write to file: "+outFileLocation)
|
||||
|
|
@ -106,8 +173,15 @@ func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk) error {
|
|||
if err != nil {
|
||||
return errors.Wrap(err, "failed to process mod zip")
|
||||
}
|
||||
defer inFile.Close()
|
||||
|
||||
if _, err := io.Copy(outFile, inFile); err != nil {
|
||||
progressInReader := &Progresser{
|
||||
Reader: inFile,
|
||||
Total: int64(file.UncompressedSize64),
|
||||
Updates: updates,
|
||||
}
|
||||
|
||||
if _, err := io.Copy(outFile, progressInReader); err != nil {
|
||||
return errors.Wrap(err, "failed to write to file: "+outFileLocation)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue