feat: parallel downloads (#43)

* feat: parallel downloads

* feat: mod extract progress using file size

* feat: pass mod version in install progress updates

* fix: only close update channels after finished sending

* chore: verbose ci tests

* fix: store mod in cache
chore: add progress logging to tests

* chore: lint

* test: add concurrent download limit

* fix: prevent concurrent map access

* chore: bump pubgrub
fix: fix race conditions

---------

Co-authored-by: Vilsol <me@vil.so>

.github/workflows/push.yaml

@@ -88,6 +88,6 @@ jobs:
         run: go generate -tags tools -x ./...
 
       - name: Test
-        run: go test ./...
+        run: go test -v ./...
         env:
           SF_DEDICATED_SERVER: ${{ github.workspace }}/SatisfactoryDedicatedServer

cfg/test_defaults.go

@@ -17,4 +17,5 @@ func SetDefaults() {
 	viper.SetDefault("dry-run", false)
 	viper.SetDefault("api-base", "https://api.ficsit.app")
 	viper.SetDefault("graphql-api", "/v2/query")
+	viper.SetDefault("concurrent-downloads", 5)
 }

cli/cache/download.go

@@ -13,34 +13,11 @@ import (
 	"github.com/satisfactorymodding/ficsit-cli/utils"
 )
 
-type Progresser struct {
+func DownloadOrCache(cacheKey string, hash string, url string, updates chan<- utils.GenericProgress, downloadSemaphore chan int) (io.ReaderAt, int64, error) {
-	io.Reader
+	if updates != nil {
-	updates chan utils.GenericUpdate
+		defer close(updates)
-	total   int64
-	running int64
-}
-
-func (pt *Progresser) Read(p []byte) (int, error) {
-	n, err := pt.Reader.Read(p)
-	pt.running += int64(n)
-
-	if err == nil {
-		if pt.updates != nil {
-			select {
-			case pt.updates <- utils.GenericUpdate{Progress: float64(pt.running) / float64(pt.total)}:
-			default:
-			}
-		}
 	}
 
-	if err == io.EOF {
-		return n, io.EOF
-	}
-
-	return n, errors.Wrap(err, "failed to read")
-}
-
-func DownloadOrCache(cacheKey string, hash string, url string, updates chan utils.GenericUpdate) (io.ReaderAt, int64, error) {
 	downloadCache := filepath.Join(viper.GetString("cache-dir"), "downloadCache")
 	if err := os.MkdirAll(downloadCache, 0o777); err != nil {
 		if !os.IsExist(err) {
@@ -82,6 +59,20 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
 		return nil, 0, errors.Wrap(err, "failed to stat file: "+location)
 	}
 
+	if updates != nil {
+		headResp, err := http.Head(url)
+		if err != nil {
+			return nil, 0, errors.Wrap(err, "failed to head: "+url)
+		}
+		defer headResp.Body.Close()
+		updates <- utils.GenericProgress{Total: headResp.ContentLength}
+	}
+
+	if downloadSemaphore != nil {
+		downloadSemaphore <- 1
+		defer func() { <-downloadSemaphore }()
+	}
+
 	out, err := os.Create(location)
 	if err != nil {
 		return nil, 0, errors.Wrap(err, "failed creating file at: "+location)
@@ -98,10 +89,10 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
 		return nil, 0, fmt.Errorf("bad status: %s on url: %s", resp.Status, url)
 	}
 
-	progresser := &Progresser{
+	progresser := &utils.Progresser{
 		Reader:  resp.Body,
-		total:   resp.ContentLength,
+		Total:   resp.ContentLength,
-		updates: updates,
+		Updates: updates,
 	}
 
 	_, err = io.Copy(out, progresser)
@@ -116,7 +107,7 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
 
 	if updates != nil {
 		select {
-		case updates <- utils.GenericUpdate{Progress: 1}:
+		case updates <- utils.GenericProgress{Completed: resp.ContentLength, Total: resp.ContentLength}:
 		default:
 		}
 	}

cli/dependency_resolver.go

@@ -9,6 +9,7 @@ import (
 	"github.com/mircearoata/pubgrub-go/pubgrub/helpers"
 	"github.com/mircearoata/pubgrub-go/pubgrub/semver"
 	"github.com/pkg/errors"
+	"github.com/puzpuzpuz/xsync/v3"
 	"github.com/spf13/viper"
 
 	"github.com/satisfactorymodding/ficsit-cli/cli/provider"
@@ -35,7 +36,7 @@ type ficsitAPISource struct {
 	provider       provider.Provider
 	lockfile       *LockFile
 	toInstall      map[string]semver.Constraint
-	modVersionInfo map[string]ficsit.ModVersionsWithDependenciesResponse
+	modVersionInfo *xsync.MapOf[string, ficsit.ModVersionsWithDependenciesResponse]
 	gameVersion    semver.Version
 	smlVersions    []ficsit.SMLVersionsSmlVersionsGetSMLVersionsSml_versionsSMLVersion
 }
@@ -74,7 +75,7 @@ func (f *ficsitAPISource) GetPackageVersions(pkg string) ([]pubgrub.PackageVersi
 	if response.Mod.Id == "" {
 		return nil, errors.Errorf("mod %s not found", pkg)
 	}
-	f.modVersionInfo[pkg] = *response
+	f.modVersionInfo.Store(pkg, *response)
 	versions := make([]pubgrub.PackageVersion, len(response.Mod.Versions))
 	for i, modVersion := range response.Mod.Versions {
 		v, err := semver.NewVersion(modVersion.Version)
@@ -145,7 +146,7 @@ func (d DependencyResolver) ResolveModDependencies(constraints map[string]string
 		gameVersion:    gameVersionSemver,
 		lockfile:       lockFile,
 		toInstall:      toInstall,
-		modVersionInfo: make(map[string]ficsit.ModVersionsWithDependenciesResponse),
+		modVersionInfo: xsync.NewMapOf[string, ficsit.ModVersionsWithDependenciesResponse](),
 	}
 
 	result, err := pubgrub.Solve(helpers.NewCachingSource(ficsitSource), rootPkg)
@@ -170,7 +171,8 @@ func (d DependencyResolver) ResolveModDependencies(constraints map[string]string
 			}
 			continue
 		}
-		versions := ficsitSource.modVersionInfo[k].Mod.Versions
+		value, _ := ficsitSource.modVersionInfo.Load(k)
+		versions := value.Mod.Versions
 		for _, ver := range versions {
 			if ver.Version == v.RawString() {
 				outputLock[k] = LockedMod{

cli/installations.go

@@ -13,6 +13,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/satisfactorymodding/ficsit-cli/cli/cache"
 	"github.com/satisfactorymodding/ficsit-cli/cli/disk"
@@ -351,14 +352,27 @@ func (i *Installation) ResolveProfile(ctx *GlobalContext) (LockFile, error) {
 	return lockfile, nil
 }
 
+type InstallUpdateType string
+
+var (
+	InstallUpdateTypeOverall     InstallUpdateType = "overall"
+	InstallUpdateTypeModDownload InstallUpdateType = "download"
+	InstallUpdateTypeModExtract  InstallUpdateType = "extract"
+	InstallUpdateTypeModComplete InstallUpdateType = "complete"
+)
+
 type InstallUpdate struct {
-	ModName          string
+	Type     InstallUpdateType
-	OverallProgress  float64
+	Item     InstallUpdateItem
-	DownloadProgress float64
+	Progress utils.GenericProgress
-	ExtractProgress  float64
 }
 
-func (i *Installation) Install(ctx *GlobalContext, updates chan InstallUpdate) error {
+type InstallUpdateItem struct {
+	Mod     string
+	Version string
+}
+
+func (i *Installation) Install(ctx *GlobalContext, updates chan<- InstallUpdate) error {
 	if err := i.Validate(ctx); err != nil {
 		return errors.Wrap(err, "failed to validate installation")
 	}
@@ -403,78 +417,65 @@ func (i *Installation) Install(ctx *GlobalContext, updates chan InstallUpdate) e
 		}
 	}
 
-	downloading := true
+	log.Info().Int("concurrency", viper.GetInt("concurrent-downloads")).Str("path", i.Path).Msg("starting installation")
-	completed := 0
 
-	var genericUpdates chan utils.GenericUpdate
+	errg := errgroup.Group{}
+	channelUsers := sync.WaitGroup{}
+	downloadSemaphore := make(chan int, viper.GetInt("concurrent-downloads"))
+	defer close(downloadSemaphore)
+
+	var modComplete chan int
 	if updates != nil {
-		var wg sync.WaitGroup
+		channelUsers.Add(1)
-		wg.Add(1)
+		modComplete = make(chan int)
-		defer wg.Wait()
+		defer close(modComplete)
-
-		genericUpdates = make(chan utils.GenericUpdate)
-		defer close(genericUpdates)
-
 		go func() {
-			defer wg.Done()
+			defer channelUsers.Done()
-
+			completed := 0
-			update := InstallUpdate{
+			for range modComplete {
-				OverallProgress:  float64(completed) / float64(len(lockfile)),
+				completed++
-				DownloadProgress: 0,
+				overallUpdate := InstallUpdate{
-				ExtractProgress:  0,
+					Type: InstallUpdateTypeOverall,
-			}
+					Progress: utils.GenericProgress{
-
+						Completed: int64(completed),
-			select {
+						Total:     int64(len(lockfile)),
-			case updates <- update:
+					},
-			default:
-			}
-
-			for up := range genericUpdates {
-				if downloading {
-					update.DownloadProgress = up.Progress
-				} else {
-					update.DownloadProgress = 1
-					update.ExtractProgress = up.Progress
-				}
-
-				if up.ModReference != nil {
-					update.ModName = *up.ModReference
-				}
-
-				update.OverallProgress = float64(completed) / float64(len(lockfile))
-
-				select {
-				case updates <- update:
-				default:
 				}
+				updates <- overallUpdate
 			}
 		}()
 	}
 
 	for modReference, version := range lockfile {
+		channelUsers.Add(1)
+		modReference := modReference
+		version := version
+		errg.Go(func() error {
+			defer channelUsers.Done()
 			// Only install if a link is provided, otherwise assume mod is already installed
 			if version.Link != "" {
-			downloading = true
+				err := downloadAndExtractMod(modReference, version.Version, version.Link, version.Hash, modsDirectory, updates, downloadSemaphore, d)
-
-			if genericUpdates != nil {
-				genericUpdates <- utils.GenericUpdate{ModReference: &modReference}
-			}
-
-			log.Info().Str("mod_reference", modReference).Str("version", version.Version).Str("link", version.Link).Msg("downloading mod")
-			reader, size, err := cache.DownloadOrCache(modReference+"_"+version.Version+".zip", version.Hash, version.Link, genericUpdates)
 				if err != nil {
-				return errors.Wrap(err, "failed to download "+modReference+" from: "+version.Link)
+					return errors.Wrapf(err, "failed to install %s@%s", modReference, version.Version)
-			}
-
-			downloading = false
-
-			log.Info().Str("mod_reference", modReference).Str("version", version.Version).Str("link", version.Link).Msg("extracting mod")
-			if err := utils.ExtractMod(reader, size, filepath.Join(modsDirectory, modReference), version.Hash, genericUpdates, d); err != nil {
-				return errors.Wrap(err, "could not extract "+modReference)
 				}
 			}
 
-		completed++
+			if modComplete != nil {
+				modComplete <- 1
+			}
+			return nil
+		})
+	}
+
+	if updates != nil {
+		go func() {
+			channelUsers.Wait()
+			close(updates)
+		}()
+	}
+
+	if err := errg.Wait(); err != nil {
+		return errors.Wrap(err, "failed to install mods")
 	}
 
 	return nil
@@ -518,6 +519,84 @@ func (i *Installation) UpdateMods(ctx *GlobalContext, mods []string) error {
 	return nil
 }
 
+func downloadAndExtractMod(modReference string, version string, link string, hash string, modsDirectory string, updates chan<- InstallUpdate, downloadSemaphore chan int, d disk.Disk) error {
+	var downloadUpdates chan utils.GenericProgress
+
+	if updates != nil {
+		// Forward the inner updates as InstallUpdates
+		downloadUpdates = make(chan utils.GenericProgress)
+
+		go func() {
+			for up := range downloadUpdates {
+				updates <- InstallUpdate{
+					Item: InstallUpdateItem{
+						Mod:     modReference,
+						Version: version,
+					},
+					Type:     InstallUpdateTypeModDownload,
+					Progress: up,
+				}
+			}
+		}()
+	}
+
+	log.Info().Str("mod_reference", modReference).Str("version", version).Str("link", link).Msg("downloading mod")
+	reader, size, err := cache.DownloadOrCache(modReference+"_"+version+".zip", hash, link, downloadUpdates, downloadSemaphore)
+	if err != nil {
+		return errors.Wrap(err, "failed to download "+modReference+" from: "+link)
+	}
+
+	var extractUpdates chan utils.GenericProgress
+
+	var wg sync.WaitGroup
+	if updates != nil {
+		// Forward the inner updates as InstallUpdates
+		extractUpdates = make(chan utils.GenericProgress)
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for up := range extractUpdates {
+				select {
+				case updates <- InstallUpdate{
+					Item: InstallUpdateItem{
+						Mod:     modReference,
+						Version: version,
+					},
+					Type:     InstallUpdateTypeModExtract,
+					Progress: up,
+				}:
+				default:
+				}
+			}
+		}()
+	}
+
+	log.Info().Str("mod_reference", modReference).Str("version", version).Str("link", link).Msg("extracting mod")
+	if err := utils.ExtractMod(reader, size, filepath.Join(modsDirectory, modReference), hash, extractUpdates, d); err != nil {
+		return errors.Wrap(err, "could not extract "+modReference)
+	}
+
+	if updates != nil {
+		select {
+		case updates <- InstallUpdate{
+			Type: InstallUpdateTypeModComplete,
+			Item: InstallUpdateItem{
+				Mod:     modReference,
+				Version: version,
+			},
+		}:
+		default:
+		}
+
+		close(extractUpdates)
+	}
+
+	wg.Wait()
+
+	return nil
+}
+
 func (i *Installation) SetProfile(ctx *GlobalContext, profile string) error {
 	found := false
 	for _, p := range ctx.Profiles.Profiles {

cli/installations_test.go

@@ -35,11 +35,11 @@ func TestAddInstallation(t *testing.T) {
 		testza.AssertNoError(t, err)
 		testza.AssertNotNil(t, installation)
 
-		err = installation.Install(ctx, nil)
+		err = installation.Install(ctx, installWatcher())
 		testza.AssertNoError(t, err)
 
 		installation.Vanilla = true
-		err = installation.Install(ctx, nil)
+		err = installation.Install(ctx, installWatcher())
 		testza.AssertNoError(t, err)
 	}
 }

cli/resolving_test.go

@@ -6,8 +6,15 @@ import (
 	"testing"
 
 	"github.com/MarvinJWendt/testza"
+	"github.com/rs/zerolog/log"
+
+	"github.com/satisfactorymodding/ficsit-cli/cfg"
 )
 
+func init() {
+	cfg.SetDefaults()
+}
+
 func profilesGetResolver() DependencyResolver {
 	ctx, err := InitCLI(false)
 	if err != nil {
@@ -17,6 +24,22 @@ func profilesGetResolver() DependencyResolver {
 	return NewDependencyResolver(ctx.Provider)
 }
 
+func installWatcher() chan<- InstallUpdate {
+	c := make(chan InstallUpdate)
+	go func() {
+		for i := range c {
+			if i.Progress.Total == i.Progress.Completed {
+				if i.Type != InstallUpdateTypeOverall {
+					log.Info().Str("mod_reference", i.Item.Mod).Str("version", i.Item.Version).Str("type", string(i.Type)).Msg("progress completed")
+				} else {
+					log.Info().Msg("overall completed")
+				}
+			}
+		}
+	}()
+	return c
+}
+
 func TestProfileResolution(t *testing.T) {
 	resolver := profilesGetResolver()
 
@@ -108,7 +131,7 @@ func TestUpdateMods(t *testing.T) {
 		err = installation.WriteLockFile(ctx, oldLockfile)
 		testza.AssertNoError(t, err)
 
-		err = installation.Install(ctx, nil)
+		err = installation.Install(ctx, installWatcher())
 		testza.AssertNoError(t, err)
 
 		lockFile, err := installation.LockFile(ctx)
@@ -126,7 +149,7 @@ func TestUpdateMods(t *testing.T) {
 		testza.AssertEqual(t, 2, len(*lockFile))
 		testza.AssertEqual(t, "1.6.6", (*lockFile)["AreaActions"].Version)
 
-		err = installation.Install(ctx, nil)
+		err = installation.Install(ctx, installWatcher())
 		testza.AssertNoError(t, err)
 	}
 }

cmd/root.go

@@ -139,6 +139,7 @@ func init() {
 	RootCmd.PersistentFlags().String("api-key", "", "API key to use when sending requests")
 
 	RootCmd.PersistentFlags().Bool("offline", false, "Whether to only use local data")
+	RootCmd.PersistentFlags().Int("concurrent-downloads", 5, "Maximum number of concurrent downloads")
 
 	_ = viper.BindPFlag("log", RootCmd.PersistentFlags().Lookup("log"))
 	_ = viper.BindPFlag("log-file", RootCmd.PersistentFlags().Lookup("log-file"))
@@ -157,4 +158,5 @@ func init() {
 	_ = viper.BindPFlag("api-key", RootCmd.PersistentFlags().Lookup("api-key"))
 
 	_ = viper.BindPFlag("offline", RootCmd.PersistentFlags().Lookup("offline"))
+	_ = viper.BindPFlag("concurrent-downloads", RootCmd.PersistentFlags().Lookup("concurrent-downloads"))
 }

go.mod

@@ -13,14 +13,16 @@ require (
 	github.com/charmbracelet/glamour v0.5.0
 	github.com/charmbracelet/lipgloss v0.6.0
 	github.com/jlaffaye/ftp v0.1.0
-	github.com/mircearoata/pubgrub-go v0.3.2
+	github.com/mircearoata/pubgrub-go v0.3.3
 	github.com/muesli/reflow v0.3.0
 	github.com/pkg/errors v0.9.1
 	github.com/pterm/pterm v0.12.67
+	github.com/puzpuzpuz/xsync/v3 v3.0.2
 	github.com/rs/zerolog v1.28.0
 	github.com/sahilm/fuzzy v0.1.0
 	github.com/spf13/cobra v1.6.0
 	github.com/spf13/viper v1.13.0
+	golang.org/x/sync v0.1.0
 )
 
 require (

go.sum

@@ -269,8 +269,8 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
 github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
 github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
 github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
-github.com/mircearoata/pubgrub-go v0.3.2 h1:AeC2bvHebii6YrfyN+AST06WiQUENKQkREA7Xoi4/HY=
+github.com/mircearoata/pubgrub-go v0.3.3 h1:XGwL8Xh5GX+mbnvWItbM/lVJxAq3NZtfUtbJ/hUf2ig=
-github.com/mircearoata/pubgrub-go v0.3.2/go.mod h1:9oWL9ZXdjFYvnGl95qiM1dTciFNx1MN8fUnG3SUwDi8=
+github.com/mircearoata/pubgrub-go v0.3.3/go.mod h1:9oWL9ZXdjFYvnGl95qiM1dTciFNx1MN8fUnG3SUwDi8=
 github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -310,6 +310,8 @@ github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5b
 github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s=
 github.com/pterm/pterm v0.12.67 h1:5iB7ajIQROYfxYD7+sFJ4+KJhFJ+xn7QOVBm4s6RUF0=
 github.com/pterm/pterm v0.12.67/go.mod h1:nFuT9ZVkkCi8o4L1dtWuYPwDQxggLh4C263qG5nTLpQ=
+github.com/puzpuzpuz/xsync/v3 v3.0.2 h1:3yESHrRFYr6xzkz61LLkvNiPFXxJEAABanTQpKbAaew=
+github.com/puzpuzpuz/xsync/v3 v3.0.2/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
@@ -493,6 +495,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

tea/scenes/apply.go

@@ -1,6 +1,8 @@
 package scenes
 
 import (
+	"sort"
+
 	"github.com/charmbracelet/bubbles/progress"
 	tea "github.com/charmbracelet/bubbletea"
 	"github.com/charmbracelet/lipgloss"
@@ -9,19 +11,23 @@ import (
 	"github.com/satisfactorymodding/ficsit-cli/cli"
 	"github.com/satisfactorymodding/ficsit-cli/tea/components"
 	"github.com/satisfactorymodding/ficsit-cli/tea/scenes/keys"
-	"github.com/satisfactorymodding/ficsit-cli/tea/utils"
+	teaUtils "github.com/satisfactorymodding/ficsit-cli/tea/utils"
+	"github.com/satisfactorymodding/ficsit-cli/utils"
 )
 
 var _ tea.Model = (*apply)(nil)
 
-type update struct {
+type modProgress struct {
+	downloadProgress utils.GenericProgress
+	extractProgress  utils.GenericProgress
+	downloading      bool
+	complete         bool
+}
+
+type status struct {
+	modProgresses   map[string]modProgress
 	installName     string
-	modName        string
+	overallProgress utils.GenericProgress
-	completed      []string
-	installTotal   int
-	installCurrent int
-	modTotal       int
-	modCurrent     int
 	done            bool
 }
 
@@ -29,11 +35,13 @@ type apply struct {
 	root           components.RootModel
 	parent         tea.Model
 	error          *components.ErrorComponent
-	updateChannel chan update
+	installChannel chan string
+	updateChannel  chan cli.InstallUpdate
+	doneChannel    chan bool
 	errorChannel   chan error
 	cancelChannel  chan bool
 	title          string
-	status        update
+	status         status
 	overall        progress.Model
 	sub            progress.Model
 	cancelled      bool
@@ -43,86 +51,46 @@ func NewApply(root components.RootModel, parent tea.Model) tea.Model {
 	overall := progress.New(progress.WithSolidFill("118"))
 	sub := progress.New(progress.WithSolidFill("202"))
 
-	updateChannel := make(chan update)
+	installChannel := make(chan string)
+	updateChannel := make(chan cli.InstallUpdate)
+	doneChannel := make(chan bool, 1)
 	errorChannel := make(chan error)
 	cancelChannel := make(chan bool, 1)
 
 	model := &apply{
 		root:    root,
 		parent:  parent,
-		title:   utils.NonListTitleStyle.MarginTop(1).MarginBottom(1).Render("Applying Changes"),
+		title:   teaUtils.NonListTitleStyle.MarginTop(1).MarginBottom(1).Render("Applying Changes"),
 		overall: overall,
 		sub:     sub,
-		status: update{
+		status: status{
-			completed: []string{},
-
 			installName: "",
-			installTotal:   100,
-			installCurrent: 0,
-
-			modName:    "",
-			modTotal:   100,
-			modCurrent: 0,
-
 			done:        false,
 		},
+		installChannel: installChannel,
 		updateChannel:  updateChannel,
+		doneChannel:    doneChannel,
 		errorChannel:   errorChannel,
 		cancelChannel:  cancelChannel,
 		cancelled:      false,
 	}
 
 	go func() {
-		result := &update{
-			completed: make([]string, 0),
-
-			installName:    "",
-			installTotal:   100,
-			installCurrent: 0,
-
-			modName:    "",
-			modTotal:   100,
-			modCurrent: 0,
-
-			done: false,
-		}
-		updateChannel <- *result
-
 		for _, installation := range root.GetGlobal().Installations.Installations {
-			result.installName = installation.Path
+			installChannel <- installation.Path
-			updateChannel <- *result
-
-			installChannel := make(chan cli.InstallUpdate)
 
+			installUpdateChannel := make(chan cli.InstallUpdate)
 			go func() {
-				for data := range installChannel {
+				for update := range installUpdateChannel {
-					result.installName = installation.Path
+					updateChannel <- update
-					result.installCurrent = int(data.OverallProgress * 100)
-
-					if data.DownloadProgress < 1 {
-						result.modName = "Downloading: " + data.ModName
-						result.modCurrent = int(data.DownloadProgress * 100)
-					} else {
-						result.modName = "Extracting: " + data.ModName
-						result.modCurrent = int(data.ExtractProgress * 100)
-					}
-
-					updateChannel <- *result
 				}
 			}()
 
-			if err := installation.Install(root.GetGlobal(), installChannel); err != nil {
+			if err := installation.Install(root.GetGlobal(), installUpdateChannel); err != nil {
 				errorChannel <- err
 				return
 			}
 
-			close(installChannel)
-
-			result.modName = ""
-			result.installTotal = 100
-			result.completed = append(result.completed, installation.Path)
-			updateChannel <- *result
-
 			stop := false
 			select {
 			case <-cancelChannel:
@@ -135,16 +103,14 @@ func NewApply(root components.RootModel, parent tea.Model) tea.Model {
 			}
 		}
 
-		result.done = true
+		doneChannel <- true
-		result.installName = ""
-		updateChannel <- *result
 	}()
 
 	return model
 }
 
 func (m apply) Init() tea.Cmd {
-	return utils.Ticker()
+	return teaUtils.Ticker()
 }
 
 func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
@@ -169,10 +135,38 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 		m.root.SetSize(msg)
 	case components.ErrorComponentTimeoutMsg:
 		m.error = nil
-	case utils.TickMsg:
+	case teaUtils.TickMsg:
 		select {
-		case newStatus := <-m.updateChannel:
+		case <-m.doneChannel:
-			m.status = newStatus
+			m.status.done = true
+			m.status.installName = ""
+			break
+		case installName := <-m.installChannel:
+			m.status.installName = installName
+			m.status.modProgresses = make(map[string]modProgress)
+			m.status.overallProgress = utils.GenericProgress{}
+			break
+		case update := <-m.updateChannel:
+			switch update.Type {
+			case cli.InstallUpdateTypeOverall:
+				m.status.overallProgress = update.Progress
+			case cli.InstallUpdateTypeModDownload:
+				m.status.modProgresses[update.Item.Mod] = modProgress{
+					downloadProgress: update.Progress,
+					downloading:      true,
+					complete:         false,
+				}
+			case cli.InstallUpdateTypeModExtract:
+				m.status.modProgresses[update.Item.Mod] = modProgress{
+					extractProgress: update.Progress,
+					downloading:     false,
+					complete:        false,
+				}
+			case cli.InstallUpdateTypeModComplete:
+				m.status.modProgresses[update.Item.Mod] = modProgress{
+					complete: true,
+				}
+			}
 			break
 		case err := <-m.errorChannel:
 			wrappedErrMessage := wrap.String(err.Error(), int(float64(m.root.Size().Width)*0.8))
@@ -183,7 +177,7 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 			// Skip if nothing there
 			break
 		}
-		return m, utils.Ticker()
+		return m, teaUtils.Ticker()
 	}
 
 	return m, nil
@@ -191,30 +185,38 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 
 func (m apply) View() string {
 	strs := make([]string, 0)
-	for _, s := range m.status.completed {
-		strs = append(strs, lipgloss.NewStyle().Foreground(lipgloss.Color("22")).Render("✓ ")+s)
-	}
 
 	if m.status.installName != "" {
-		marginTop := 0
+		strs = append(strs, lipgloss.NewStyle().Render(m.status.installName))
-		if len(m.status.completed) > 0 {
+		strs = append(strs, lipgloss.NewStyle().MarginBottom(1).Render(m.overall.ViewAs(m.status.overallProgress.Percentage())))
-			marginTop = 1
 	}
 
-		strs = append(strs, lipgloss.NewStyle().MarginTop(marginTop).Render(m.status.installName))
+	keys := make([]string, 0)
-		strs = append(strs, m.overall.ViewAs(float64(m.status.installCurrent)/float64(m.status.installTotal)))
+	for k := range m.status.modProgresses {
+		keys = append(keys, k)
 	}
+	sort.Strings(keys)
 
-	if m.status.modName != "" {
+	for _, modReference := range keys {
-		strs = append(strs, lipgloss.NewStyle().MarginTop(1).Render(m.status.modName))
+		p := m.status.modProgresses[modReference]
-		strs = append(strs, m.sub.ViewAs(float64(m.status.modCurrent)/float64(m.status.modTotal)))
+		if p.complete {
+			strs = append(strs, lipgloss.NewStyle().Foreground(lipgloss.Color("22")).Render("✓ ")+modReference)
+		} else {
+			if p.downloading {
+				strs = append(strs, lipgloss.NewStyle().Render(modReference+" (Downloading)"))
+				strs = append(strs, m.sub.ViewAs(p.downloadProgress.Percentage()))
+			} else {
+				strs = append(strs, lipgloss.NewStyle().Render(modReference+" (Extracting)"))
+				strs = append(strs, m.sub.ViewAs(p.extractProgress.Percentage()))
+			}
+		}
 	}
 
 	if m.status.done {
 		if m.cancelled {
-			strs = append(strs, utils.LabelStyle.Copy().Foreground(lipgloss.Color("196")).Padding(0).Margin(1).Render("Cancelled! Press Enter to return"))
+			strs = append(strs, teaUtils.LabelStyle.Copy().Foreground(lipgloss.Color("196")).Padding(0).Margin(1).Render("Cancelled! Press Enter to return"))
 		} else {
-			strs = append(strs, utils.LabelStyle.Copy().Padding(0).Margin(1).Render("Done! Press Enter to return"))
+			strs = append(strs, teaUtils.LabelStyle.Copy().Padding(0).Margin(1).Render("Done! Press Enter to return"))
 		}
 	}
 

utils/io.go

@@ -7,15 +7,51 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"sync"
+	"sync/atomic"
 
 	"github.com/pkg/errors"
 
 	"github.com/satisfactorymodding/ficsit-cli/cli/disk"
 )
 
-type GenericUpdate struct {
+type GenericProgress struct {
-	ModReference *string
+	Completed int64
-	Progress     float64
+	Total     int64
+}
+
+func (gp GenericProgress) Percentage() float64 {
+	if gp.Total == 0 {
+		return 0
+	}
+	return float64(gp.Completed) / float64(gp.Total)
+}
+
+type Progresser struct {
+	io.Reader
+	Updates chan<- GenericProgress
+	Total   int64
+	Running int64
+}
+
+func (pt *Progresser) Read(p []byte) (int, error) {
+	n, err := pt.Reader.Read(p)
+	pt.Running += int64(n)
+
+	if err == nil {
+		if pt.Updates != nil {
+			select {
+			case pt.Updates <- GenericProgress{Completed: pt.Running, Total: pt.Total}:
+			default:
+			}
+		}
+	}
+
+	if err == io.EOF {
+		return n, io.EOF
+	}
+
+	return n, errors.Wrap(err, "failed to read")
 }
 
 func SHA256Data(f io.Reader) (string, error) {
@@ -27,7 +63,7 @@ func SHA256Data(f io.Reader) (string, error) {
 	return hex.EncodeToString(h.Sum(nil)), nil
 }
 
-func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates chan GenericUpdate, d disk.Disk) error {
+func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates chan<- GenericProgress, d disk.Disk) error {
 	hashFile := filepath.Join(location, ".smm")
 	hashBytes, err := d.Read(hashFile)
 	if err != nil {
@@ -59,7 +95,24 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
 		return errors.Wrap(err, "failed to read file as zip")
 	}
 
-	for i, file := range reader.File {
+	totalSize := int64(0)
+
+	for _, file := range reader.File {
+		totalSize += int64(file.UncompressedSize64)
+	}
+
+	totalExtracted := int64(0)
+	totalExtractedPtr := &totalExtracted
+
+	channelUsers := sync.WaitGroup{}
+
+	if updates != nil {
+		defer func() {
+			channelUsers.Wait()
+		}()
+	}
+
+	for _, file := range reader.File {
 		if !file.FileInfo().IsDir() {
 			outFileLocation := filepath.Join(location, file.Name)
 
@@ -67,16 +120,26 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
 				return errors.Wrap(err, "failed to create mod directory: "+location)
 			}
 
-			if err := writeZipFile(outFileLocation, file, d); err != nil {
+			var fileUpdates chan GenericProgress
-				return err
+			if updates != nil {
+				fileUpdates = make(chan GenericProgress)
+				channelUsers.Add(1)
+				go func() {
+					defer channelUsers.Done()
+					for fileUpdate := range fileUpdates {
+						updates <- GenericProgress{
+							Completed: atomic.LoadInt64(totalExtractedPtr) + fileUpdate.Completed,
+							Total:     totalSize,
 						}
 					}
+				}()
+			}
 
-		if updates != nil {
+			if err := writeZipFile(outFileLocation, file, d, fileUpdates); err != nil {
-			select {
+				return err
-			case updates <- GenericUpdate{Progress: float64(i) / float64(len(reader.File)-1)}:
-			default:
 			}
+
+			atomic.AddInt64(totalExtractedPtr, int64(file.UncompressedSize64))
 		}
 	}
 
@@ -86,7 +149,7 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
 
 	if updates != nil {
 		select {
-		case updates <- GenericUpdate{Progress: 1}:
+		case updates <- GenericProgress{Completed: totalSize, Total: totalSize}:
 		default:
 		}
 	}
@@ -94,7 +157,11 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
 	return nil
 }
 
-func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk) error {
+func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk, updates chan<- GenericProgress) error {
+	if updates != nil {
+		defer close(updates)
+	}
+
 	outFile, err := d.Open(outFileLocation, os.O_CREATE|os.O_RDWR)
 	if err != nil {
 		return errors.Wrap(err, "failed to write to file: "+outFileLocation)
@@ -106,8 +173,15 @@ func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to process mod zip")
 	}
+	defer inFile.Close()
 
-	if _, err := io.Copy(outFile, inFile); err != nil {
+	progressInReader := &Progresser{
+		Reader:  inFile,
+		Total:   int64(file.UncompressedSize64),
+		Updates: updates,
+	}
+
+	if _, err := io.Copy(outFile, progressInReader); err != nil {
 		return errors.Wrap(err, "failed to write to file: "+outFileLocation)
 	}