From 2dda29ce480c9c528d052f48052ece48cfe753aa Mon Sep 17 00:00:00 2001 From: Nicolas JUHEL Date: Fri, 11 Sep 2020 13:49:40 +0200 Subject: [PATCH] PKG Progress : add complex bar features, refactor & optimize, fix some errors & linter --- progress/bar.go | 111 ++++++++++++++------ progress/progress.go | 242 +++++++++++++++++++++++++++++++------------ 2 files changed, 251 insertions(+), 102 deletions(-) diff --git a/progress/bar.go b/progress/bar.go index a1c40db..711b86b 100644 --- a/progress/bar.go +++ b/progress/bar.go @@ -26,47 +26,49 @@ package progress import ( - "context" + "time" + "github.com/nabbar/golib/errors" + "github.com/nabbar/golib/semaphore" "github.com/vbauerster/mpb/v5" - - . "github.com/nabbar/golib/errors" - - sem "github.com/nabbar/golib/semaphore" ) type bar struct { - u bool + i time.Time + s semaphore.Sem t int64 - b *mpb.Bar - s sem.Sem + u bool + w bool } type Bar interface { Current() int64 Completed() bool - Increment(n int) - Refill(amount int64) + Reset(total, current int64) + ResetDefined(current int64) + Done() - NewWorker() Error + Increment(n int) + Increment64(n int64) + + NewWorker() errors.Error NewWorkerTry() bool DeferWorker() DeferMain(dropBar bool) - WaitAll() Error - Context() context.Context - Cancel() + WaitAll() errors.Error GetBarMPB() *mpb.Bar } -func newBar(b *mpb.Bar, s sem.Sem, total int64) Bar { +func newBar(b *mpb.Bar, s semaphore.Sem, total int64, isModeUnic bool) Bar { return &bar{ u: total > 0, t: total, b: b, s: s, + w: isModeUnic, } } @@ -83,47 +85,88 @@ func (b bar) Completed() bool { } func (b *bar) Increment(n int) { - if n == 0 { - n = 1 + if n > 0 { + b.b.IncrBy(n) + + if b.i != semaphore.EmptyTime() { + b.b.DecoratorEwmaUpdate(time.Since(b.i)) + } } - b.b.IncrBy(n) + + b.i = time.Now() } -func (b *bar) Refill(amount int64) { - b.b.SetRefill(amount) +func (b *bar) Increment64(n int64) { + if n > 0 { + b.b.IncrInt64(n) + + if b.i != semaphore.EmptyTime() { + b.b.DecoratorEwmaUpdate(time.Since(b.i)) + } + } + + b.i = time.Now() } -func (b *bar) NewWorker() Error { +func (b *bar) ResetDefined(current int64) { + if current >= b.t { + b.b.SetTotal(b.t, true) + b.b.SetRefill(b.t) + } else { + b.b.SetTotal(b.t, false) + b.b.SetRefill(current) + } +} + +func (b *bar) Reset(total, current int64) { + b.u = total > 0 + b.t = total + b.ResetDefined(current) +} + +func (b *bar) Done() { + b.b.SetRefill(b.t) + b.b.SetTotal(b.t, true) +} + +func (b *bar) NewWorker() errors.Error { if !b.u { b.t++ b.b.SetTotal(b.t, false) } - return b.s.NewWorker() + if !b.w { + return b.s.NewWorker() + } + + return nil } func (b *bar) NewWorkerTry() bool { - return b.s.NewWorkerTry() + + if !b.w { + return b.s.NewWorkerTry() + } + + return false } func (b *bar) DeferWorker() { - b.b.Increment() + b.Increment(1) b.s.DeferWorker() } func (b *bar) DeferMain(dropBar bool) { b.b.Abort(dropBar) - b.s.DeferMain() + if !b.w { + b.s.DeferMain() + } } -func (b *bar) WaitAll() Error { - return b.s.WaitAll() -} +func (b *bar) WaitAll() errors.Error { + if !b.w { + return b.s.WaitAll() + } -func (b bar) Context() context.Context { - return b.s.Context() -} - -func (b bar) Cancel() { - b.s.Cancel() + return nil } diff --git a/progress/progress.go b/progress/progress.go index b6bfdfb..a525ed1 100644 --- a/progress/progress.go +++ b/progress/progress.go @@ -27,8 +27,8 @@ package progress import ( "context" - "time" + "github.com/nabbar/golib/errors" "github.com/nabbar/golib/semaphore" "github.com/vbauerster/mpb/v5" "github.com/vbauerster/mpb/v5/decor" @@ -62,35 +62,45 @@ func GetDefaultMessageDone() string { type progressBar struct { mpb *mpb.Progress ctx context.Context - cnl context.CancelFunc - sTimeOut time.Duration sMaxSimul int + sem semaphore.Sem } type ProgressBar interface { GetMPB() *mpb.Progress - - GetContext() context.Context + SetMaxThread(maxSimultaneous int) SetContext(ctx context.Context) - GetCancel() context.CancelFunc - SetCancel(cancel context.CancelFunc) + UnicProcessInit() + UnicProcessWait() errors.Error + UnicProcessNewWorker() errors.Error + UnicProcessDeferWorker() + UnicProcessDefer() - SetSemaphoreOption(maxSimultaneous int, timeout time.Duration) + NewBar(total int64, options ...mpb.BarOption) Bar - NewBar(parent context.Context, total int64, options ...mpb.BarOption) Bar - NewBarSimpleETA(name string) Bar + NewBarETA(name string, total int64, job string, parent Bar) Bar + NewBarCounter(name string, total int64, job string, parent Bar) Bar + NewBarKBits(name string, total int64, job string, parent Bar) Bar + + NewBarSimpleETA(name string, total int64) Bar NewBarSimpleCounter(name string, total int64) Bar + NewBarSimpleKBits(name string, total int64) Bar } -func NewProgressBar(timeout time.Duration, deadline time.Time, parent context.Context, options ...mpb.ContainerOption) ProgressBar { - x, c := semaphore.GetContext(timeout, deadline, parent) +func NewProgressBar(options ...mpb.ContainerOption) ProgressBar { + return NewProgressBarWithContext(context.Background(), options...) +} + +func NewProgressBarWithContext(ctx context.Context, options ...mpb.ContainerOption) ProgressBar { + if ctx == nil { + ctx = context.Background() + } return &progressBar{ mpb: mpb.New(options...), - ctx: x, - cnl: c, - sTimeOut: timeout, + ctx: ctx, + sem: nil, sMaxSimul: semaphore.GetMaxSimultaneous(), } } @@ -99,77 +109,173 @@ func (p *progressBar) GetMPB() *mpb.Progress { return p.mpb } -func (p *progressBar) SetSemaphoreOption(maxSimultaneous int, timeout time.Duration) { +func (p *progressBar) SetMaxThread(maxSimultaneous int) { p.sMaxSimul = maxSimultaneous - p.sTimeOut = timeout } -func (p *progressBar) NewBar(parent context.Context, total int64, options ...mpb.BarOption) Bar { - if parent == nil { - parent = p.ctx - } +func (p *progressBar) UnicProcessInit() { + p.sem = p.semaphore() +} +func (p *progressBar) UnicProcessWait() errors.Error { + if p.sem != nil { + return p.sem.WaitAll() + } + return nil +} + +func (p *progressBar) UnicProcessNewWorker() errors.Error { + if p.sem != nil { + return p.sem.NewWorker() + } + return nil +} + +func (p *progressBar) UnicProcessDeferWorker() { + if p.sem != nil { + p.sem.DeferWorker() + } +} + +func (p *progressBar) UnicProcessDefer() { + if p.sem != nil { + p.sem.DeferMain() + } +} + +func (p *progressBar) semaphore() semaphore.Sem { + return semaphore.NewSemaphoreWithContext(p.ctx, p.sMaxSimul) +} + +func (p *progressBar) NewBar(total int64, options ...mpb.BarOption) Bar { return newBar( p.mpb.AddBar(0, options...), - semaphore.NewSemaphore(p.sMaxSimul, p.sTimeOut, semaphore.GetEmptyTime(), parent), + p.semaphore(), total, + p.sem != nil, ) } -func (p *progressBar) NewBarSimpleETA(name string) Bar { - return newBar( - p.mpb.AddBar(0, - mpb.BarStyle(defaultStyle), - mpb.PrependDecorators( - // display our name with one space on the right - decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}), - // replace ETA decorator with "done" message, OnComplete event - decor.OnComplete( - // nolint: gomnd - decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: 4}), defaultMessageDone, - ), - ), - mpb.AppendDecorators(decor.Percentage()), - ), - semaphore.NewSemaphore(p.sMaxSimul, p.sTimeOut, semaphore.GetEmptyTime(), p.ctx), - 0, - ) +func (p *progressBar) NewBarSimpleETA(name string, total int64) Bar { + return p.NewBarETA(name, total, "", nil) } func (p *progressBar) NewBarSimpleCounter(name string, total int64) Bar { - return newBar( - p.mpb.AddBar(total, - mpb.BarStyle(defaultStyle), - mpb.PrependDecorators( - // display our name with one space on the right - decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}), - // use counter (no ETA) - decor.CountersNoUnit("[%d / %d] ", decor.WCSyncWidth), - // replace ETA decorator with "done" message, OnComplete event - decor.OnComplete( - // nolint: gomnd - decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: 4}), defaultMessageDone, - ), - ), - mpb.AppendDecorators(decor.Percentage()), - ), - semaphore.NewSemaphore(p.sMaxSimul, p.sTimeOut, semaphore.GetEmptyTime(), p.ctx), - total, + return p.NewBarCounter(name, total, "", nil) +} + +func (p *progressBar) NewBarSimpleKBits(name string, total int64) Bar { + return p.NewBarKBits(name, total, "", nil) +} + +func (p *progressBar) NewBarETA(name string, total int64, job string, parent Bar) Bar { + if parent != nil && job != "" { + return newBar(p.addBarJob(total, name, job, nil, nil, parent.GetBarMPB()), p.semaphore(), total, p.sem != nil) + } else { + return newBar(p.addBarSimple(total, name, nil, nil), p.semaphore(), total, p.sem != nil) + } +} + +func (p *progressBar) NewBarCounter(name string, total int64, job string, parent Bar) Bar { + d := decor.CountersNoUnit("[%d / %d] ", decor.WCSyncWidth) + if parent != nil && job != "" { + return newBar(p.addBarJob(total, name, job, d, nil, parent.GetBarMPB()), p.semaphore(), total, p.sem != nil) + } else { + return newBar(p.addBarSimple(total, name, d, nil), p.semaphore(), total, p.sem != nil) + } +} + +func (p *progressBar) NewBarKBits(name string, total int64, job string, parent Bar) Bar { + //nolint #gomnd + d := decor.Counters(decor.UnitKiB, "% .2f / % .2f", decor.WC{W: 20, C: decor.DextraSpace}) + a := []decor.Decorator{ + //nolint #gomnd + decor.Percentage(decor.WC{W: 5, C: 0}), + decor.Name(" | "), + //nolint #gomnd + decor.EwmaSpeed(decor.UnitKiB, "% .2f", 60), + } + + if parent != nil && job != "" { + return newBar(p.addBarJob(total, name, job, d, a, parent.GetBarMPB()), p.semaphore(), total, p.sem != nil) + } else { + return newBar(p.addBarSimple(total, name, d, a), p.semaphore(), total, p.sem != nil) + } +} + +func (p *progressBar) addBarSimple(total int64, name string, counter decor.Decorator, pct []decor.Decorator) *mpb.Bar { + pr := make([]decor.Decorator, 0) + // display our name with one space on the right + pr = append(pr, decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight})) + if counter != nil { + // use counter (no ETA) + pr = append(pr, counter) + } + //nolint #gomnd + pr = append(pr, decor.Name(" ", decor.WC{W: 3, C: decor.DidentRight | decor.DextraSpace})) + // replace ETA decorator with "done" message, OnComplete event + pr = append(pr, decor.OnComplete( + // nolint: gomnd + decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: len(defaultMessageDone) + 1, C: 0}), defaultMessageDone, + )) + + if pct == nil { + pct = make([]decor.Decorator, 0) + //nolint #gomnd + pct = append(pct, decor.Percentage(decor.WC{W: 5, C: 0})) + } + + return p.mpb.AddBar(total, + mpb.BarStyle(defaultStyle), + mpb.BarFillerClearOnComplete(), + mpb.PrependDecorators(pr...), + mpb.AppendDecorators(pct...), ) } -func (p *progressBar) GetContext() context.Context { - return p.ctx +func (p *progressBar) addBarJob(total int64, name, job string, counter decor.Decorator, pct []decor.Decorator, bar *mpb.Bar) *mpb.Bar { + pr := make([]decor.Decorator, 0) + // display our name with one space on the right + pr = append(pr, decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight})) + // display our job task with one space on the right + pr = append(pr, decor.Name(job, decor.WC{W: len(job) + 1, C: decor.DidentRight | decor.DextraSpace})) + if counter != nil { + // use counter (no ETA) + pr = append(pr, counter) + } + //nolint #gomnd + pr = append(pr, decor.Name(" ", decor.WC{W: 3, C: decor.DidentRight | decor.DextraSpace})) + if bar != nil { + pr = append(pr, decor.OnComplete( + // replace ETA decorator with "done" message, OnComplete event + decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: len(defaultMessageDone) + 1, C: 0}), defaultMessageDone, + )) + } + + if pct == nil { + pct = make([]decor.Decorator, 0) + //nolint #gomnd + pct = append(pct, decor.Percentage(decor.WC{W: 5, C: 0})) + } + + if bar == nil { + return p.mpb.AddBar(total, + mpb.BarStyle(defaultStyle), + mpb.BarFillerClearOnComplete(), + mpb.PrependDecorators(pr...), + mpb.AppendDecorators(pct...), + ) + } else { + return p.mpb.AddBar(total, + mpb.BarStyle(defaultStyle), + mpb.BarQueueAfter(bar), + mpb.BarFillerClearOnComplete(), + mpb.PrependDecorators(pr...), + mpb.AppendDecorators(pct...), + ) + } } func (p *progressBar) SetContext(ctx context.Context) { p.ctx = ctx } - -func (p *progressBar) GetCancel() context.CancelFunc { - return p.cnl -} - -func (p *progressBar) SetCancel(cancel context.CancelFunc) { - p.cnl = cancel -}