feat: parallel downloads (#43)

* feat: parallel downloads

* feat: mod extract progress using file size

* feat: pass mod version in install progress updates

* fix: only close update channels after finished sending

* chore: verbose ci tests

* fix: store mod in cache
chore: add progress logging to tests

* chore: lint

* test: add concurrent download limit

* fix: prevent concurrent map access

* chore: bump pubgrub
fix: fix race conditions

---------

Co-authored-by: Vilsol <me@vil.so>
This commit is contained in:
mircearoata 2023-12-07 00:39:34 +01:00 committed by GitHub
parent a192a63c82
commit 6088d1e8eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 401 additions and 222 deletions

View file

@ -88,6 +88,6 @@ jobs:
run: go generate -tags tools -x ./...
- name: Test
run: go test ./...
run: go test -v ./...
env:
SF_DEDICATED_SERVER: ${{ github.workspace }}/SatisfactoryDedicatedServer

View file

@ -17,4 +17,5 @@ func SetDefaults() {
viper.SetDefault("dry-run", false)
viper.SetDefault("api-base", "https://api.ficsit.app")
viper.SetDefault("graphql-api", "/v2/query")
viper.SetDefault("concurrent-downloads", 5)
}

51
cli/cache/download.go vendored
View file

@ -13,34 +13,11 @@ import (
"github.com/satisfactorymodding/ficsit-cli/utils"
)
type Progresser struct {
io.Reader
updates chan utils.GenericUpdate
total int64
running int64
}
func (pt *Progresser) Read(p []byte) (int, error) {
n, err := pt.Reader.Read(p)
pt.running += int64(n)
if err == nil {
if pt.updates != nil {
select {
case pt.updates <- utils.GenericUpdate{Progress: float64(pt.running) / float64(pt.total)}:
default:
}
}
func DownloadOrCache(cacheKey string, hash string, url string, updates chan<- utils.GenericProgress, downloadSemaphore chan int) (io.ReaderAt, int64, error) {
if updates != nil {
defer close(updates)
}
if err == io.EOF {
return n, io.EOF
}
return n, errors.Wrap(err, "failed to read")
}
func DownloadOrCache(cacheKey string, hash string, url string, updates chan utils.GenericUpdate) (io.ReaderAt, int64, error) {
downloadCache := filepath.Join(viper.GetString("cache-dir"), "downloadCache")
if err := os.MkdirAll(downloadCache, 0o777); err != nil {
if !os.IsExist(err) {
@ -82,6 +59,20 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
return nil, 0, errors.Wrap(err, "failed to stat file: "+location)
}
if updates != nil {
headResp, err := http.Head(url)
if err != nil {
return nil, 0, errors.Wrap(err, "failed to head: "+url)
}
defer headResp.Body.Close()
updates <- utils.GenericProgress{Total: headResp.ContentLength}
}
if downloadSemaphore != nil {
downloadSemaphore <- 1
defer func() { <-downloadSemaphore }()
}
out, err := os.Create(location)
if err != nil {
return nil, 0, errors.Wrap(err, "failed creating file at: "+location)
@ -98,10 +89,10 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
return nil, 0, fmt.Errorf("bad status: %s on url: %s", resp.Status, url)
}
progresser := &Progresser{
progresser := &utils.Progresser{
Reader: resp.Body,
total: resp.ContentLength,
updates: updates,
Total: resp.ContentLength,
Updates: updates,
}
_, err = io.Copy(out, progresser)
@ -116,7 +107,7 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan util
if updates != nil {
select {
case updates <- utils.GenericUpdate{Progress: 1}:
case updates <- utils.GenericProgress{Completed: resp.ContentLength, Total: resp.ContentLength}:
default:
}
}

View file

@ -9,6 +9,7 @@ import (
"github.com/mircearoata/pubgrub-go/pubgrub/helpers"
"github.com/mircearoata/pubgrub-go/pubgrub/semver"
"github.com/pkg/errors"
"github.com/puzpuzpuz/xsync/v3"
"github.com/spf13/viper"
"github.com/satisfactorymodding/ficsit-cli/cli/provider"
@ -35,7 +36,7 @@ type ficsitAPISource struct {
provider provider.Provider
lockfile *LockFile
toInstall map[string]semver.Constraint
modVersionInfo map[string]ficsit.ModVersionsWithDependenciesResponse
modVersionInfo *xsync.MapOf[string, ficsit.ModVersionsWithDependenciesResponse]
gameVersion semver.Version
smlVersions []ficsit.SMLVersionsSmlVersionsGetSMLVersionsSml_versionsSMLVersion
}
@ -74,7 +75,7 @@ func (f *ficsitAPISource) GetPackageVersions(pkg string) ([]pubgrub.PackageVersi
if response.Mod.Id == "" {
return nil, errors.Errorf("mod %s not found", pkg)
}
f.modVersionInfo[pkg] = *response
f.modVersionInfo.Store(pkg, *response)
versions := make([]pubgrub.PackageVersion, len(response.Mod.Versions))
for i, modVersion := range response.Mod.Versions {
v, err := semver.NewVersion(modVersion.Version)
@ -145,7 +146,7 @@ func (d DependencyResolver) ResolveModDependencies(constraints map[string]string
gameVersion: gameVersionSemver,
lockfile: lockFile,
toInstall: toInstall,
modVersionInfo: make(map[string]ficsit.ModVersionsWithDependenciesResponse),
modVersionInfo: xsync.NewMapOf[string, ficsit.ModVersionsWithDependenciesResponse](),
}
result, err := pubgrub.Solve(helpers.NewCachingSource(ficsitSource), rootPkg)
@ -170,7 +171,8 @@ func (d DependencyResolver) ResolveModDependencies(constraints map[string]string
}
continue
}
versions := ficsitSource.modVersionInfo[k].Mod.Versions
value, _ := ficsitSource.modVersionInfo.Load(k)
versions := value.Mod.Versions
for _, ver := range versions {
if ver.Version == v.RawString() {
outputLock[k] = LockedMod{

View file

@ -13,6 +13,7 @@ import (
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/spf13/viper"
"golang.org/x/sync/errgroup"
"github.com/satisfactorymodding/ficsit-cli/cli/cache"
"github.com/satisfactorymodding/ficsit-cli/cli/disk"
@ -351,14 +352,27 @@ func (i *Installation) ResolveProfile(ctx *GlobalContext) (LockFile, error) {
return lockfile, nil
}
type InstallUpdateType string
var (
InstallUpdateTypeOverall InstallUpdateType = "overall"
InstallUpdateTypeModDownload InstallUpdateType = "download"
InstallUpdateTypeModExtract InstallUpdateType = "extract"
InstallUpdateTypeModComplete InstallUpdateType = "complete"
)
type InstallUpdate struct {
ModName string
OverallProgress float64
DownloadProgress float64
ExtractProgress float64
Type InstallUpdateType
Item InstallUpdateItem
Progress utils.GenericProgress
}
func (i *Installation) Install(ctx *GlobalContext, updates chan InstallUpdate) error {
type InstallUpdateItem struct {
Mod string
Version string
}
func (i *Installation) Install(ctx *GlobalContext, updates chan<- InstallUpdate) error {
if err := i.Validate(ctx); err != nil {
return errors.Wrap(err, "failed to validate installation")
}
@ -403,78 +417,65 @@ func (i *Installation) Install(ctx *GlobalContext, updates chan InstallUpdate) e
}
}
downloading := true
completed := 0
log.Info().Int("concurrency", viper.GetInt("concurrent-downloads")).Str("path", i.Path).Msg("starting installation")
var genericUpdates chan utils.GenericUpdate
errg := errgroup.Group{}
channelUsers := sync.WaitGroup{}
downloadSemaphore := make(chan int, viper.GetInt("concurrent-downloads"))
defer close(downloadSemaphore)
var modComplete chan int
if updates != nil {
var wg sync.WaitGroup
wg.Add(1)
defer wg.Wait()
genericUpdates = make(chan utils.GenericUpdate)
defer close(genericUpdates)
channelUsers.Add(1)
modComplete = make(chan int)
defer close(modComplete)
go func() {
defer wg.Done()
update := InstallUpdate{
OverallProgress: float64(completed) / float64(len(lockfile)),
DownloadProgress: 0,
ExtractProgress: 0,
}
select {
case updates <- update:
default:
}
for up := range genericUpdates {
if downloading {
update.DownloadProgress = up.Progress
} else {
update.DownloadProgress = 1
update.ExtractProgress = up.Progress
}
if up.ModReference != nil {
update.ModName = *up.ModReference
}
update.OverallProgress = float64(completed) / float64(len(lockfile))
select {
case updates <- update:
default:
defer channelUsers.Done()
completed := 0
for range modComplete {
completed++
overallUpdate := InstallUpdate{
Type: InstallUpdateTypeOverall,
Progress: utils.GenericProgress{
Completed: int64(completed),
Total: int64(len(lockfile)),
},
}
updates <- overallUpdate
}
}()
}
for modReference, version := range lockfile {
channelUsers.Add(1)
modReference := modReference
version := version
errg.Go(func() error {
defer channelUsers.Done()
// Only install if a link is provided, otherwise assume mod is already installed
if version.Link != "" {
downloading = true
if genericUpdates != nil {
genericUpdates <- utils.GenericUpdate{ModReference: &modReference}
}
log.Info().Str("mod_reference", modReference).Str("version", version.Version).Str("link", version.Link).Msg("downloading mod")
reader, size, err := cache.DownloadOrCache(modReference+"_"+version.Version+".zip", version.Hash, version.Link, genericUpdates)
err := downloadAndExtractMod(modReference, version.Version, version.Link, version.Hash, modsDirectory, updates, downloadSemaphore, d)
if err != nil {
return errors.Wrap(err, "failed to download "+modReference+" from: "+version.Link)
}
downloading = false
log.Info().Str("mod_reference", modReference).Str("version", version.Version).Str("link", version.Link).Msg("extracting mod")
if err := utils.ExtractMod(reader, size, filepath.Join(modsDirectory, modReference), version.Hash, genericUpdates, d); err != nil {
return errors.Wrap(err, "could not extract "+modReference)
return errors.Wrapf(err, "failed to install %s@%s", modReference, version.Version)
}
}
completed++
if modComplete != nil {
modComplete <- 1
}
return nil
})
}
if updates != nil {
go func() {
channelUsers.Wait()
close(updates)
}()
}
if err := errg.Wait(); err != nil {
return errors.Wrap(err, "failed to install mods")
}
return nil
@ -518,6 +519,84 @@ func (i *Installation) UpdateMods(ctx *GlobalContext, mods []string) error {
return nil
}
func downloadAndExtractMod(modReference string, version string, link string, hash string, modsDirectory string, updates chan<- InstallUpdate, downloadSemaphore chan int, d disk.Disk) error {
var downloadUpdates chan utils.GenericProgress
if updates != nil {
// Forward the inner updates as InstallUpdates
downloadUpdates = make(chan utils.GenericProgress)
go func() {
for up := range downloadUpdates {
updates <- InstallUpdate{
Item: InstallUpdateItem{
Mod: modReference,
Version: version,
},
Type: InstallUpdateTypeModDownload,
Progress: up,
}
}
}()
}
log.Info().Str("mod_reference", modReference).Str("version", version).Str("link", link).Msg("downloading mod")
reader, size, err := cache.DownloadOrCache(modReference+"_"+version+".zip", hash, link, downloadUpdates, downloadSemaphore)
if err != nil {
return errors.Wrap(err, "failed to download "+modReference+" from: "+link)
}
var extractUpdates chan utils.GenericProgress
var wg sync.WaitGroup
if updates != nil {
// Forward the inner updates as InstallUpdates
extractUpdates = make(chan utils.GenericProgress)
wg.Add(1)
go func() {
defer wg.Done()
for up := range extractUpdates {
select {
case updates <- InstallUpdate{
Item: InstallUpdateItem{
Mod: modReference,
Version: version,
},
Type: InstallUpdateTypeModExtract,
Progress: up,
}:
default:
}
}
}()
}
log.Info().Str("mod_reference", modReference).Str("version", version).Str("link", link).Msg("extracting mod")
if err := utils.ExtractMod(reader, size, filepath.Join(modsDirectory, modReference), hash, extractUpdates, d); err != nil {
return errors.Wrap(err, "could not extract "+modReference)
}
if updates != nil {
select {
case updates <- InstallUpdate{
Type: InstallUpdateTypeModComplete,
Item: InstallUpdateItem{
Mod: modReference,
Version: version,
},
}:
default:
}
close(extractUpdates)
}
wg.Wait()
return nil
}
func (i *Installation) SetProfile(ctx *GlobalContext, profile string) error {
found := false
for _, p := range ctx.Profiles.Profiles {

View file

@ -35,11 +35,11 @@ func TestAddInstallation(t *testing.T) {
testza.AssertNoError(t, err)
testza.AssertNotNil(t, installation)
err = installation.Install(ctx, nil)
err = installation.Install(ctx, installWatcher())
testza.AssertNoError(t, err)
installation.Vanilla = true
err = installation.Install(ctx, nil)
err = installation.Install(ctx, installWatcher())
testza.AssertNoError(t, err)
}
}

View file

@ -6,8 +6,15 @@ import (
"testing"
"github.com/MarvinJWendt/testza"
"github.com/rs/zerolog/log"
"github.com/satisfactorymodding/ficsit-cli/cfg"
)
func init() {
cfg.SetDefaults()
}
func profilesGetResolver() DependencyResolver {
ctx, err := InitCLI(false)
if err != nil {
@ -17,6 +24,22 @@ func profilesGetResolver() DependencyResolver {
return NewDependencyResolver(ctx.Provider)
}
func installWatcher() chan<- InstallUpdate {
c := make(chan InstallUpdate)
go func() {
for i := range c {
if i.Progress.Total == i.Progress.Completed {
if i.Type != InstallUpdateTypeOverall {
log.Info().Str("mod_reference", i.Item.Mod).Str("version", i.Item.Version).Str("type", string(i.Type)).Msg("progress completed")
} else {
log.Info().Msg("overall completed")
}
}
}
}()
return c
}
func TestProfileResolution(t *testing.T) {
resolver := profilesGetResolver()
@ -108,7 +131,7 @@ func TestUpdateMods(t *testing.T) {
err = installation.WriteLockFile(ctx, oldLockfile)
testza.AssertNoError(t, err)
err = installation.Install(ctx, nil)
err = installation.Install(ctx, installWatcher())
testza.AssertNoError(t, err)
lockFile, err := installation.LockFile(ctx)
@ -126,7 +149,7 @@ func TestUpdateMods(t *testing.T) {
testza.AssertEqual(t, 2, len(*lockFile))
testza.AssertEqual(t, "1.6.6", (*lockFile)["AreaActions"].Version)
err = installation.Install(ctx, nil)
err = installation.Install(ctx, installWatcher())
testza.AssertNoError(t, err)
}
}

View file

@ -139,6 +139,7 @@ func init() {
RootCmd.PersistentFlags().String("api-key", "", "API key to use when sending requests")
RootCmd.PersistentFlags().Bool("offline", false, "Whether to only use local data")
RootCmd.PersistentFlags().Int("concurrent-downloads", 5, "Maximum number of concurrent downloads")
_ = viper.BindPFlag("log", RootCmd.PersistentFlags().Lookup("log"))
_ = viper.BindPFlag("log-file", RootCmd.PersistentFlags().Lookup("log-file"))
@ -157,4 +158,5 @@ func init() {
_ = viper.BindPFlag("api-key", RootCmd.PersistentFlags().Lookup("api-key"))
_ = viper.BindPFlag("offline", RootCmd.PersistentFlags().Lookup("offline"))
_ = viper.BindPFlag("concurrent-downloads", RootCmd.PersistentFlags().Lookup("concurrent-downloads"))
}

4
go.mod
View file

@ -13,14 +13,16 @@ require (
github.com/charmbracelet/glamour v0.5.0
github.com/charmbracelet/lipgloss v0.6.0
github.com/jlaffaye/ftp v0.1.0
github.com/mircearoata/pubgrub-go v0.3.2
github.com/mircearoata/pubgrub-go v0.3.3
github.com/muesli/reflow v0.3.0
github.com/pkg/errors v0.9.1
github.com/pterm/pterm v0.12.67
github.com/puzpuzpuz/xsync/v3 v3.0.2
github.com/rs/zerolog v1.28.0
github.com/sahilm/fuzzy v0.1.0
github.com/spf13/cobra v1.6.0
github.com/spf13/viper v1.13.0
golang.org/x/sync v0.1.0
)
require (

7
go.sum
View file

@ -269,8 +269,8 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
github.com/mircearoata/pubgrub-go v0.3.2 h1:AeC2bvHebii6YrfyN+AST06WiQUENKQkREA7Xoi4/HY=
github.com/mircearoata/pubgrub-go v0.3.2/go.mod h1:9oWL9ZXdjFYvnGl95qiM1dTciFNx1MN8fUnG3SUwDi8=
github.com/mircearoata/pubgrub-go v0.3.3 h1:XGwL8Xh5GX+mbnvWItbM/lVJxAq3NZtfUtbJ/hUf2ig=
github.com/mircearoata/pubgrub-go v0.3.3/go.mod h1:9oWL9ZXdjFYvnGl95qiM1dTciFNx1MN8fUnG3SUwDi8=
github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@ -310,6 +310,8 @@ github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5b
github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s=
github.com/pterm/pterm v0.12.67 h1:5iB7ajIQROYfxYD7+sFJ4+KJhFJ+xn7QOVBm4s6RUF0=
github.com/pterm/pterm v0.12.67/go.mod h1:nFuT9ZVkkCi8o4L1dtWuYPwDQxggLh4C263qG5nTLpQ=
github.com/puzpuzpuz/xsync/v3 v3.0.2 h1:3yESHrRFYr6xzkz61LLkvNiPFXxJEAABanTQpKbAaew=
github.com/puzpuzpuz/xsync/v3 v3.0.2/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
@ -493,6 +495,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

View file

@ -1,6 +1,8 @@
package scenes
import (
"sort"
"github.com/charmbracelet/bubbles/progress"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
@ -9,19 +11,23 @@ import (
"github.com/satisfactorymodding/ficsit-cli/cli"
"github.com/satisfactorymodding/ficsit-cli/tea/components"
"github.com/satisfactorymodding/ficsit-cli/tea/scenes/keys"
"github.com/satisfactorymodding/ficsit-cli/tea/utils"
teaUtils "github.com/satisfactorymodding/ficsit-cli/tea/utils"
"github.com/satisfactorymodding/ficsit-cli/utils"
)
var _ tea.Model = (*apply)(nil)
type update struct {
type modProgress struct {
downloadProgress utils.GenericProgress
extractProgress utils.GenericProgress
downloading bool
complete bool
}
type status struct {
modProgresses map[string]modProgress
installName string
modName string
completed []string
installTotal int
installCurrent int
modTotal int
modCurrent int
overallProgress utils.GenericProgress
done bool
}
@ -29,11 +35,13 @@ type apply struct {
root components.RootModel
parent tea.Model
error *components.ErrorComponent
updateChannel chan update
installChannel chan string
updateChannel chan cli.InstallUpdate
doneChannel chan bool
errorChannel chan error
cancelChannel chan bool
title string
status update
status status
overall progress.Model
sub progress.Model
cancelled bool
@ -43,86 +51,46 @@ func NewApply(root components.RootModel, parent tea.Model) tea.Model {
overall := progress.New(progress.WithSolidFill("118"))
sub := progress.New(progress.WithSolidFill("202"))
updateChannel := make(chan update)
installChannel := make(chan string)
updateChannel := make(chan cli.InstallUpdate)
doneChannel := make(chan bool, 1)
errorChannel := make(chan error)
cancelChannel := make(chan bool, 1)
model := &apply{
root: root,
parent: parent,
title: utils.NonListTitleStyle.MarginTop(1).MarginBottom(1).Render("Applying Changes"),
title: teaUtils.NonListTitleStyle.MarginTop(1).MarginBottom(1).Render("Applying Changes"),
overall: overall,
sub: sub,
status: update{
completed: []string{},
status: status{
installName: "",
installTotal: 100,
installCurrent: 0,
modName: "",
modTotal: 100,
modCurrent: 0,
done: false,
},
installChannel: installChannel,
updateChannel: updateChannel,
doneChannel: doneChannel,
errorChannel: errorChannel,
cancelChannel: cancelChannel,
cancelled: false,
}
go func() {
result := &update{
completed: make([]string, 0),
installName: "",
installTotal: 100,
installCurrent: 0,
modName: "",
modTotal: 100,
modCurrent: 0,
done: false,
}
updateChannel <- *result
for _, installation := range root.GetGlobal().Installations.Installations {
result.installName = installation.Path
updateChannel <- *result
installChannel := make(chan cli.InstallUpdate)
installChannel <- installation.Path
installUpdateChannel := make(chan cli.InstallUpdate)
go func() {
for data := range installChannel {
result.installName = installation.Path
result.installCurrent = int(data.OverallProgress * 100)
if data.DownloadProgress < 1 {
result.modName = "Downloading: " + data.ModName
result.modCurrent = int(data.DownloadProgress * 100)
} else {
result.modName = "Extracting: " + data.ModName
result.modCurrent = int(data.ExtractProgress * 100)
}
updateChannel <- *result
for update := range installUpdateChannel {
updateChannel <- update
}
}()
if err := installation.Install(root.GetGlobal(), installChannel); err != nil {
if err := installation.Install(root.GetGlobal(), installUpdateChannel); err != nil {
errorChannel <- err
return
}
close(installChannel)
result.modName = ""
result.installTotal = 100
result.completed = append(result.completed, installation.Path)
updateChannel <- *result
stop := false
select {
case <-cancelChannel:
@ -135,16 +103,14 @@ func NewApply(root components.RootModel, parent tea.Model) tea.Model {
}
}
result.done = true
result.installName = ""
updateChannel <- *result
doneChannel <- true
}()
return model
}
func (m apply) Init() tea.Cmd {
return utils.Ticker()
return teaUtils.Ticker()
}
func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
@ -169,10 +135,38 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.root.SetSize(msg)
case components.ErrorComponentTimeoutMsg:
m.error = nil
case utils.TickMsg:
case teaUtils.TickMsg:
select {
case newStatus := <-m.updateChannel:
m.status = newStatus
case <-m.doneChannel:
m.status.done = true
m.status.installName = ""
break
case installName := <-m.installChannel:
m.status.installName = installName
m.status.modProgresses = make(map[string]modProgress)
m.status.overallProgress = utils.GenericProgress{}
break
case update := <-m.updateChannel:
switch update.Type {
case cli.InstallUpdateTypeOverall:
m.status.overallProgress = update.Progress
case cli.InstallUpdateTypeModDownload:
m.status.modProgresses[update.Item.Mod] = modProgress{
downloadProgress: update.Progress,
downloading: true,
complete: false,
}
case cli.InstallUpdateTypeModExtract:
m.status.modProgresses[update.Item.Mod] = modProgress{
extractProgress: update.Progress,
downloading: false,
complete: false,
}
case cli.InstallUpdateTypeModComplete:
m.status.modProgresses[update.Item.Mod] = modProgress{
complete: true,
}
}
break
case err := <-m.errorChannel:
wrappedErrMessage := wrap.String(err.Error(), int(float64(m.root.Size().Width)*0.8))
@ -183,7 +177,7 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
// Skip if nothing there
break
}
return m, utils.Ticker()
return m, teaUtils.Ticker()
}
return m, nil
@ -191,30 +185,38 @@ func (m apply) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
func (m apply) View() string {
strs := make([]string, 0)
for _, s := range m.status.completed {
strs = append(strs, lipgloss.NewStyle().Foreground(lipgloss.Color("22")).Render("✓ ")+s)
}
if m.status.installName != "" {
marginTop := 0
if len(m.status.completed) > 0 {
marginTop = 1
strs = append(strs, lipgloss.NewStyle().Render(m.status.installName))
strs = append(strs, lipgloss.NewStyle().MarginBottom(1).Render(m.overall.ViewAs(m.status.overallProgress.Percentage())))
}
strs = append(strs, lipgloss.NewStyle().MarginTop(marginTop).Render(m.status.installName))
strs = append(strs, m.overall.ViewAs(float64(m.status.installCurrent)/float64(m.status.installTotal)))
keys := make([]string, 0)
for k := range m.status.modProgresses {
keys = append(keys, k)
}
sort.Strings(keys)
if m.status.modName != "" {
strs = append(strs, lipgloss.NewStyle().MarginTop(1).Render(m.status.modName))
strs = append(strs, m.sub.ViewAs(float64(m.status.modCurrent)/float64(m.status.modTotal)))
for _, modReference := range keys {
p := m.status.modProgresses[modReference]
if p.complete {
strs = append(strs, lipgloss.NewStyle().Foreground(lipgloss.Color("22")).Render("✓ ")+modReference)
} else {
if p.downloading {
strs = append(strs, lipgloss.NewStyle().Render(modReference+" (Downloading)"))
strs = append(strs, m.sub.ViewAs(p.downloadProgress.Percentage()))
} else {
strs = append(strs, lipgloss.NewStyle().Render(modReference+" (Extracting)"))
strs = append(strs, m.sub.ViewAs(p.extractProgress.Percentage()))
}
}
}
if m.status.done {
if m.cancelled {
strs = append(strs, utils.LabelStyle.Copy().Foreground(lipgloss.Color("196")).Padding(0).Margin(1).Render("Cancelled! Press Enter to return"))
strs = append(strs, teaUtils.LabelStyle.Copy().Foreground(lipgloss.Color("196")).Padding(0).Margin(1).Render("Cancelled! Press Enter to return"))
} else {
strs = append(strs, utils.LabelStyle.Copy().Padding(0).Margin(1).Render("Done! Press Enter to return"))
strs = append(strs, teaUtils.LabelStyle.Copy().Padding(0).Margin(1).Render("Done! Press Enter to return"))
}
}

View file

@ -7,15 +7,51 @@ import (
"io"
"os"
"path/filepath"
"sync"
"sync/atomic"
"github.com/pkg/errors"
"github.com/satisfactorymodding/ficsit-cli/cli/disk"
)
type GenericUpdate struct {
ModReference *string
Progress float64
type GenericProgress struct {
Completed int64
Total int64
}
func (gp GenericProgress) Percentage() float64 {
if gp.Total == 0 {
return 0
}
return float64(gp.Completed) / float64(gp.Total)
}
type Progresser struct {
io.Reader
Updates chan<- GenericProgress
Total int64
Running int64
}
func (pt *Progresser) Read(p []byte) (int, error) {
n, err := pt.Reader.Read(p)
pt.Running += int64(n)
if err == nil {
if pt.Updates != nil {
select {
case pt.Updates <- GenericProgress{Completed: pt.Running, Total: pt.Total}:
default:
}
}
}
if err == io.EOF {
return n, io.EOF
}
return n, errors.Wrap(err, "failed to read")
}
func SHA256Data(f io.Reader) (string, error) {
@ -27,7 +63,7 @@ func SHA256Data(f io.Reader) (string, error) {
return hex.EncodeToString(h.Sum(nil)), nil
}
func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates chan GenericUpdate, d disk.Disk) error {
func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates chan<- GenericProgress, d disk.Disk) error {
hashFile := filepath.Join(location, ".smm")
hashBytes, err := d.Read(hashFile)
if err != nil {
@ -59,7 +95,24 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
return errors.Wrap(err, "failed to read file as zip")
}
for i, file := range reader.File {
totalSize := int64(0)
for _, file := range reader.File {
totalSize += int64(file.UncompressedSize64)
}
totalExtracted := int64(0)
totalExtractedPtr := &totalExtracted
channelUsers := sync.WaitGroup{}
if updates != nil {
defer func() {
channelUsers.Wait()
}()
}
for _, file := range reader.File {
if !file.FileInfo().IsDir() {
outFileLocation := filepath.Join(location, file.Name)
@ -67,16 +120,26 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
return errors.Wrap(err, "failed to create mod directory: "+location)
}
if err := writeZipFile(outFileLocation, file, d); err != nil {
return err
var fileUpdates chan GenericProgress
if updates != nil {
fileUpdates = make(chan GenericProgress)
channelUsers.Add(1)
go func() {
defer channelUsers.Done()
for fileUpdate := range fileUpdates {
updates <- GenericProgress{
Completed: atomic.LoadInt64(totalExtractedPtr) + fileUpdate.Completed,
Total: totalSize,
}
}
}()
}
if updates != nil {
select {
case updates <- GenericUpdate{Progress: float64(i) / float64(len(reader.File)-1)}:
default:
if err := writeZipFile(outFileLocation, file, d, fileUpdates); err != nil {
return err
}
atomic.AddInt64(totalExtractedPtr, int64(file.UncompressedSize64))
}
}
@ -86,7 +149,7 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
if updates != nil {
select {
case updates <- GenericUpdate{Progress: 1}:
case updates <- GenericProgress{Completed: totalSize, Total: totalSize}:
default:
}
}
@ -94,7 +157,11 @@ func ExtractMod(f io.ReaderAt, size int64, location string, hash string, updates
return nil
}
func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk) error {
func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk, updates chan<- GenericProgress) error {
if updates != nil {
defer close(updates)
}
outFile, err := d.Open(outFileLocation, os.O_CREATE|os.O_RDWR)
if err != nil {
return errors.Wrap(err, "failed to write to file: "+outFileLocation)
@ -106,8 +173,15 @@ func writeZipFile(outFileLocation string, file *zip.File, d disk.Disk) error {
if err != nil {
return errors.Wrap(err, "failed to process mod zip")
}
defer inFile.Close()
if _, err := io.Copy(outFile, inFile); err != nil {
progressInReader := &Progresser{
Reader: inFile,
Total: int64(file.UncompressedSize64),
Updates: updates,
}
if _, err := io.Copy(outFile, progressInReader); err != nil {
return errors.Wrap(err, "failed to write to file: "+outFileLocation)
}