zishuo/golib
1
0
Fork 0
mirror of https://github.com/nabbar/golib.git synced 2025-09-27 04:06:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

Package AWS:

Browse Source 
- rework MultipartUpload process & helper
- update test to use lib size
- update object multipart to use new helper

Package IO Utils :
- add truncate & sync to FileProgress
- fix error on open file mode for FileProgress

Package Console :
- fix interface used for color buffer

Package Cobra :
- add function to print message on write config to use custom message instead of internal message. If the function is not set, the default message will be print.

Other:
- fix golangci-lint config to remove crazy linter (use only golang group compliance linter)
- bump dependencies
This commit is contained in:
nabbar
2023-05-22 16:23:23 +02:00
parent 7316c249e2
commit 1ab80f338e
22 changed files with 1286 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
.golangci.yml
View File

@@ -82,7 +82,7 @@ linters:
disable: disable:
# - bodyclose # - bodyclose
# - contextcheck # - contextcheck
# - cyclop - cyclop
- deadcode - deadcode
# - errname # - errname
# - errorlint # - errorlint
@@ -90,12 +90,12 @@ linters:
- exhaustivestruct - exhaustivestruct
- exhaustruct - exhaustruct
# - forbidigo # - forbidigo
# - funlen - funlen
# - gci # - gci
# - gochecknoglobals # - gochecknoglobals
# - gochecknoinits # - gochecknoinits
# - gocognit # - gocognit
# - gocritic - gocritic
# - gocyclo # - gocyclo
# - godot # - godot
# - godox # - godox
@@ -106,16 +106,16 @@ linters:
- ifshort - ifshort
# - interfacebloat # - interfacebloat
- interfacer - interfacer
# - ireturn - ireturn
# - lll # - lll
- maligned - maligned
# - nakedret # - nakedret
# - nestif - nestif
# - nilerr # - nilerr
# - nlreturn # - nlreturn
# - noctx # - noctx
# - nolintlint - nolintlint
# - nonamedreturns - nonamedreturns
- nosnakecase - nosnakecase
# - revive # - revive
# - rowserrcheck # - rowserrcheck
@@ -129,5 +129,5 @@ linters:
- varnamelen - varnamelen
# - wastedassign # - wastedassign
# - whitespace # - whitespace
# - wrapcheck - wrapcheck
# - wsl - wsl

5
aws/aws_suite_test.go
View File

@@ -31,6 +31,7 @@ import (
"crypto/rand" "crypto/rand"
"errors" "errors"
"fmt" "fmt"
libsiz "github.com/nabbar/golib/size"
"io/ioutil" "io/ioutil"
"net" "net"
"net/http" "net/http"
@@ -240,8 +241,8 @@ func WaitMinio(host string) bool {
return err == nil return err == nil
} }
func randContent(size int) *bytes.Buffer { func randContent(size libsiz.Size) *bytes.Buffer {
p := make([]byte, size) p := make([]byte, size.Int64())
_, err := rand.Read(p) _, err := rand.Read(p)

3
aws/bucket_test.go
View File

@@ -26,6 +26,7 @@
package aws_test package aws_test
import ( import (
libsiz "github.com/nabbar/golib/size"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
) )
@@ -52,7 +53,7 @@ var _ = Describe("Bucket", func() {
It("Must succeed", func() { It("Must succeed", func() {
var ( var (
err error err error
rnd = randContent(64 * 1024) rnd = randContent(10 * libsiz.SizeMega)
) )
err = cli.Object().MultipartPut("object", rnd) err = cli.Object().MultipartPut("object", rnd)

121
aws/helper/partSize.go
View File

@@ -1,121 +0,0 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package helper
import (
"errors"
"io"
"strings"
libsiz "github.com/nabbar/golib/size"
sdkaws "github.com/aws/aws-sdk-go-v2/aws"
sdktps "github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type ReaderPartSize interface {
io.Reader
NextPart(eTag *string)
CurrPart() int32
CompPart() *sdktps.CompletedMultipartUpload
IeOEF() bool
}
func NewReaderPartSize(rd io.Reader, p libsiz.Size) ReaderPartSize {
return &readerPartSize{
b: rd,
p: p.Int64(),
i: 1,
j: 0,
e: false,
c: nil,
}
}
type readerPartSize struct {
// buffer
b io.Reader
// partsize
p int64
// partNumber
i int64
// current part counter
j int64
// Is EOF
e bool
// complete part slice
c *sdktps.CompletedMultipartUpload
}
func (r *readerPartSize) NextPart(eTag *string) {
if r.c == nil {
r.c = &sdktps.CompletedMultipartUpload{
Parts: nil,
}
}
if r.c.Parts == nil {
r.c.Parts = make([]sdktps.CompletedPart, 0)
}
r.c.Parts = append(r.c.Parts, sdktps.CompletedPart{
ETag: sdkaws.String(strings.Replace(*eTag, "\"", "", -1)),
PartNumber: int32(r.i),
})
r.i++
r.j = 0
}
func (r readerPartSize) CurrPart() int32 {
return int32(r.i)
}
func (r readerPartSize) CompPart() *sdktps.CompletedMultipartUpload {
return r.c
}
func (r readerPartSize) IeOEF() bool {
return r.e
}
func (r *readerPartSize) Read(p []byte) (n int, err error) {
if r.e || r.j >= r.p {
return 0, io.EOF
}
if len(p) > int(r.p-r.j) {
p = make([]byte, int(r.p-r.j))
}
n, e := r.b.Read(p)
if errors.Is(e, io.EOF) {
r.e = true
}
return n, e
}

37
aws/multipart/errors.go Normal file
View File

@@ -0,0 +1,37 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package multipart
import "fmt"
var (
ErrInvalidInstance = fmt.Errorf("invalid instance")
ErrInvalidClient = fmt.Errorf("invalid aws S3 client")
ErrInvalidResponse = fmt.Errorf("invalid aws S3 response")
ErrInvalidUploadID = fmt.Errorf("invalid aws s3 MPU Upload ID")
ErrInvalidTMPFile = fmt.Errorf("invalid working or temporary file")
ErrWorkingPartFileExceedSize = fmt.Errorf("working or temporary file used exceed the aws S3 size limits")
)

114
aws/multipart/interface.go Normal file
View File

@@ -0,0 +1,114 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package multipart
import (
"fmt"
"io"
"math"
"sync"
sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
libctx "github.com/nabbar/golib/context"
libsiz "github.com/nabbar/golib/size"
)
const (
DefaultPartSize = 5 * libsiz.SizeMega
MaxPartSize = 5 * libsiz.SizeGiga
MaxObjectSize = 5 * libsiz.SizeTera
MaxNumberPart int32 = 10000
)
type FuncClientS3 func() *sdksss.Client
type MultiPart interface {
io.WriteCloser
RegisterContext(fct libctx.FuncContext)
RegisterClientS3(fct FuncClientS3)
RegisterMultipartID(id string)
RegisterWorkingFile(file string, truncate bool) error
RegisterFuncOnPushPart(fct func(eTag string, e error))
RegisterFuncOnAbort(fct func(nPart int, obj string, e error))
RegisterFuncOnComplete(fct func(nPart int, obj string, e error))
StartMPU() error
StopMPU(abort bool) error
AddPart(r io.Reader) (n int64, e error)
SendPart() error
CurrentSizePart() int64
AddToPart(p []byte) (n int, e error)
RegisterPart(etag string)
IsStarted() bool
Counter() int32
CounterLeft() int32
}
func New(partSize libsiz.Size, object string, bucket string) MultiPart {
return &mpu{
m: sync.RWMutex{},
c: nil,
s: partSize,
i: "",
o: object,
b: bucket,
n: 0,
}
}
func GetOptimalPartSize(objectSize, partSize libsiz.Size) (libsiz.Size, error) {
var (
lim = math.MaxFloat64
nbr int64
prt int64
obj int64
)
if partSize <= DefaultPartSize {
prt = DefaultPartSize.Int64()
} else {
prt = partSize.Int64()
}
obj = objectSize.Int64()
if obj > (int64(MaxNumberPart) * MaxPartSize.Int64()) {
return 0, fmt.Errorf("object size need exceed the maximum number of part with the maximum size of part")
} else if objectSize > MaxObjectSize {
return 0, fmt.Errorf("object size is over allowed maximum size of object")
} else if uint64(obj) > uint64(lim) || uint64(prt) > uint64(lim) || uint64(obj/prt) > uint64(lim) {
return GetOptimalPartSize(objectSize, libsiz.SizeFromInt64(prt*2))
} else if nbr = int64(math.Ceil(float64(obj) / float64(prt))); nbr > int64(MaxNumberPart) {
return GetOptimalPartSize(objectSize, libsiz.SizeFromInt64(prt*2))
} else if prt > MaxPartSize.Int64() {
return GetOptimalPartSize(objectSize, libsiz.SizeFromInt64((prt/4)*3))
}
return libsiz.SizeFromInt64(prt), nil
}

34
aws/multipart/io.go Normal file
View File

@@ -0,0 +1,34 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package multipart
func (m *mpu) Write(p []byte) (n int, err error) {
return m.AddToPart(p)
}
func (m *mpu) Close() error {
return m.StopMPU(false)
}

286
aws/multipart/model.go Normal file
View File

@@ -0,0 +1,286 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package multipart
import (
"context"
"fmt"
"path/filepath"
"sync"
sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
sdktyp "github.com/aws/aws-sdk-go-v2/service/s3/types"
libctx "github.com/nabbar/golib/context"
libiot "github.com/nabbar/golib/ioutils"
libsiz "github.com/nabbar/golib/size"
)
type mpu struct {
m sync.RWMutex
x libctx.FuncContext
c FuncClientS3
s libsiz.Size // part size
i string // upload id
b string // bucket name
o string // object name
n int32 // part counter
l []sdktyp.CompletedPart // slice of sent part to prepare complete MPU
w libiot.FileProgress // working file or temporary file
// trigger function
fc func(nPart int, obj string, e error) // on complete
fp func(eTag string, e error) // on push part
fa func(nPart int, obj string, e error) // on abort
}
func (m *mpu) RegisterContext(fct libctx.FuncContext) {
if m == nil {
return
}
m.m.Lock()
defer m.m.Unlock()
m.x = fct
}
func (m *mpu) getContext() context.Context {
if m == nil {
return context.Background()
}
m.m.RLock()
defer m.m.RUnlock()
if m.x == nil {
return context.Background()
} else if x := m.x(); x == nil {
return context.Background()
} else {
return x
}
}
func (m *mpu) RegisterClientS3(fct FuncClientS3) {
if m == nil {
return
}
m.m.Lock()
defer m.m.Unlock()
m.c = fct
}
func (m *mpu) getClient() *sdksss.Client {
if m == nil {
return nil
}
m.m.RLock()
defer m.m.RUnlock()
if m.c == nil {
return nil
} else if c := m.c(); c == nil {
return nil
} else {
return c
}
}
func (m *mpu) RegisterMultipartID(id string) {
if m == nil {
return
}
m.m.Lock()
defer m.m.Unlock()
m.i = id
}
func (m *mpu) getMultipartID() string {
if m == nil {
return ""
}
m.m.RLock()
defer m.m.RUnlock()
return m.i
}
func (m *mpu) RegisterWorkingFile(file string, truncate bool) error {
if m == nil {
return ErrInvalidInstance
}
m.m.Lock()
defer m.m.Unlock()
var e error
if m.w != nil {
m.m.Unlock()
if e = m.CheckSend(true, false); e != nil {
return e
}
m.m.Lock()
_ = m.w.Close()
m.w = nil
}
m.w, e = libiot.NewFileProgressPathWrite(filepath.Clean(file), true, truncate, 0600)
if e != nil {
return e
} else if truncate {
return m.w.Truncate(0)
}
return nil
}
func (m *mpu) getWorkingFile() (libiot.FileProgress, error) {
if m == nil {
return nil, ErrInvalidInstance
}
m.m.RLock()
defer m.m.RUnlock()
if m.w != nil {
return m.w, nil
}
m.m.RUnlock()
e := m.setTempWorkingFile()
m.m.RLock()
if e != nil {
return nil, e
} else if m.w == nil {
return nil, ErrInvalidTMPFile
}
return m.w, nil
}
func (m *mpu) setTempWorkingFile() error {
if m == nil {
return ErrInvalidInstance
}
m.m.Lock()
defer m.m.Unlock()
var e error
m.w, e = libiot.NewFileProgressTemp()
return e
}
func (m *mpu) closeWorkingFile() error {
if m == nil {
return nil
}
m.m.Lock()
defer m.m.Unlock()
if m.w == nil {
return nil
}
var e error
e = m.w.Truncate(0)
if er := m.w.Close(); er != nil {
if e != nil {
e = fmt.Errorf("%v, %v", e, er)
} else {
e = er
}
}
m.w = nil
return e
}
func (m *mpu) getPartSize() libsiz.Size {
if m == nil {
return DefaultPartSize
}
m.m.RLock()
defer m.m.RUnlock()
if m.s < 1 {
return DefaultPartSize
}
return m.s
}
func (m *mpu) setPartSize(s libsiz.Size) {
if m == nil {
return
}
m.m.Lock()
defer m.m.Unlock()
if s < 1 {
s = DefaultPartSize
}
m.s = s
}
func (m *mpu) getObject() string {
if m == nil {
return ""
}
m.m.RLock()
defer m.m.RUnlock()
return m.o
}
func (m *mpu) getBucket() string {
if m == nil {
return ""
}
m.m.RLock()
defer m.m.RUnlock()
return m.b
}

305
aws/multipart/part.go Normal file
View File

@@ -0,0 +1,305 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package multipart
import (
/* #nosec */
//nolint #nosec
"crypto/md5"
"encoding/base64"
"fmt"
"io"
"strings"
sdkaws "github.com/aws/aws-sdk-go-v2/aws"
sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
sdktyp "github.com/aws/aws-sdk-go-v2/service/s3/types"
libiot "github.com/nabbar/golib/ioutils"
libsiz "github.com/nabbar/golib/size"
)
func (m *mpu) getPartList() []sdktyp.CompletedPart {
if m == nil {
return make([]sdktyp.CompletedPart, 0)
}
m.m.RLock()
defer m.m.RUnlock()
if len(m.l) < 1 {
return make([]sdktyp.CompletedPart, 0)
}
return m.l
}
func (m *mpu) Counter() int32 {
if m == nil {
return 0
}
m.m.RLock()
defer m.m.RUnlock()
return m.n
}
func (m *mpu) CounterLeft() int32 {
if m == nil {
return 0
}
m.m.RLock()
defer m.m.RUnlock()
if m.n >= MaxNumberPart {
return 0
}
return MaxNumberPart - m.n
}
func (m *mpu) RegisterPart(etag string) {
if m == nil {
return
}
m.m.Lock()
defer m.m.Unlock()
if len(m.l) < 1 {
m.l = make([]sdktyp.CompletedPart, 0)
}
m.n++
m.l = append(m.l, sdktyp.CompletedPart{
ETag: sdkaws.String(strings.Replace(etag, "\"", "", -1)),
PartNumber: m.n,
})
}
func (m *mpu) AddPart(r io.Reader) (n int64, e error) {
if m == nil {
return 0, ErrInvalidInstance
}
var (
cli *sdksss.Client
res *sdksss.UploadPartOutput
tmp libiot.FileProgress
ctx = m.getContext()
obj = m.getObject()
bck = m.getBucket()
mid = m.getMultipartID()
/* #nosec */
//nolint #nosec
hsh = md5.New()
)
if cli = m.getClient(); cli == nil {
return 0, ErrInvalidClient
} else if tmp, e = libiot.NewFileProgressTemp(); e != nil {
return 0, e
} else if tmp == nil {
return 0, ErrInvalidTMPFile
} else {
defer func() {
if tmp != nil {
_ = tmp.Close()
}
}()
}
if n, e = io.Copy(tmp, r); e != nil || n < 1 {
return n, e
} else if _, e = tmp.Seek(0, io.SeekStart); e != nil {
return 0, e
} else if _, e = tmp.WriteTo(hsh); e != nil {
return 0, e
} else if _, e = tmp.Seek(0, io.SeekStart); e != nil {
return 0, e
}
res, e = cli.UploadPart(ctx, &sdksss.UploadPartInput{
Bucket: sdkaws.String(bck),
Key: sdkaws.String(obj),
UploadId: sdkaws.String(mid),
PartNumber: m.Counter() + 1,
ContentLength: n,
Body: tmp,
RequestPayer: sdktyp.RequestPayerRequester,
ContentMD5: sdkaws.String(base64.StdEncoding.EncodeToString(hsh.Sum(nil))),
})
if e != nil {
m.callFuncOnPushPart("", e)
return 0, e
} else if res == nil || res.ETag == nil || len(*res.ETag) < 1 {
m.callFuncOnPushPart("", ErrInvalidResponse)
return 0, ErrInvalidResponse
} else {
t := *res.ETag
m.callFuncOnPushPart(t, nil)
m.RegisterPart(t)
}
return n, nil
}
func (m *mpu) AddToPart(p []byte) (n int, e error) {
var (
tmp libiot.FileProgress
)
if tmp, e = m.getWorkingFile(); e != nil {
return 0, e
} else if tmp == nil {
return 0, ErrInvalidTMPFile
}
for len(p) > 0 {
var (
r []byte
i int
s int64
siz = m.getPartSize().Int64()
)
if _, e = tmp.Seek(0, io.SeekStart); e != nil {
return n, e
} else if s, e = tmp.SizeToEOF(); e != nil {
return n, e
} else if _, e = tmp.Seek(0, io.SeekEnd); e != nil {
return n, e
} else if s > 0 && s >= siz {
if e = m.CheckSend(false, false); e != nil {
return n, e
}
continue
} else if s > 0 && s < siz {
siz -= s
}
if int64(len(p)) > siz {
r = p[:siz]
p = p[siz:]
} else {
r = p
p = nil
}
if i, e = tmp.Write(r); e != nil {
return n, e
} else if i != len(r) {
return n, fmt.Errorf("write a wrong number of byte")
} else if e = m.CheckSend(false, false); e != nil {
return n, e
} else {
n += len(r)
}
}
return n, nil
}
func (m *mpu) SendPart() error {
return m.CheckSend(true, false)
}
func (m *mpu) CurrentSizePart() int64 {
var (
e error
s int64
tmp libiot.FileProgress
)
if tmp, e = m.getWorkingFile(); e != nil {
return 0
} else if tmp == nil {
return 0
} else if _, e = tmp.Seek(0, io.SeekStart); e != nil {
return 0
} else {
s, e = tmp.SizeToEOF()
_, _ = tmp.Seek(0, io.SeekEnd)
return s
}
}
func (m *mpu) CheckSend(force, close bool) error {
var (
err error
siz int64
prt = m.getPartSize()
tmp libiot.FileProgress
)
if tmp, err = m.getWorkingFile(); err != nil {
return err
} else if tmp == nil {
return ErrInvalidTMPFile
} else if _, err = tmp.Seek(0, io.SeekStart); err != nil {
return err
} else if siz, err = tmp.SizeToEOF(); err != nil {
return err
} else if siz < prt.Int64() && !force {
return nil
} else if siz == 0 {
return nil
} else if siz > int64(MaxObjectSize) {
return ErrWorkingPartFileExceedSize
} else if close && m.Counter() < 1 && siz < DefaultPartSize.Int64() {
return nil
} else if _, err = m.sendPart(siz, tmp); err != nil {
return err
} else if err = tmp.Truncate(0); err != nil {
return err
} else if err = tmp.Sync(); err != nil {
return err
} else {
return nil
}
}
func (m *mpu) sendPart(siz int64, body io.Reader) (int64, error) {
var (
err error
prt = m.getPartSize()
)
if prt, err = GetOptimalPartSize(libsiz.SizeFromInt64(siz), prt); err != nil {
return 0, err
} else if prt != m.getPartSize() {
old := m.getPartSize()
m.setPartSize(prt)
defer func() {
m.setPartSize(old)
}()
}
return m.AddPart(body)
}

92
aws/multipart/start.go Normal file
View File

@@ -0,0 +1,92 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package multipart
import (
"mime"
"path/filepath"
sdkaws "github.com/aws/aws-sdk-go-v2/aws"
sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
)
func (m *mpu) IsStarted() bool {
if cli := m.getClient(); cli == nil {
return false
} else if len(m.getMultipartID()) < 1 {
return false
}
return true
}
func (m *mpu) getMimeType() string {
if t := mime.TypeByExtension(filepath.Ext(m.getObject())); t == "" {
return "application/octet-stream"
} else {
return t
}
}
func (m *mpu) StartMPU() error {
if m == nil {
return ErrInvalidInstance
}
var (
cli *sdksss.Client
res *sdksss.CreateMultipartUploadOutput
err error
tpe = m.getMimeType()
ctx = m.getContext()
obj = m.getObject()
bck = m.getBucket()
)
if cli = m.getClient(); cli == nil {
return ErrInvalidClient
}
res, err = cli.CreateMultipartUpload(ctx, &sdksss.CreateMultipartUploadInput{
Key: sdkaws.String(obj),
Bucket: sdkaws.String(bck),
ContentType: sdkaws.String(tpe),
})
if err != nil {
return err
} else if res == nil {
return ErrInvalidResponse
} else if res.UploadId == nil || len(*res.UploadId) < 1 {
return ErrInvalidResponse
}
m.m.Lock()
defer m.m.Unlock()
m.i = *res.UploadId
return nil
}

189
aws/multipart/stop.go Normal file
View File

@@ -0,0 +1,189 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package multipart
import (
"io"
sdkaws "github.com/aws/aws-sdk-go-v2/aws"
sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
sdktyp "github.com/aws/aws-sdk-go-v2/service/s3/types"
libiot "github.com/nabbar/golib/ioutils"
)
func (m *mpu) StopMPU(abort bool) error {
if m == nil {
return ErrInvalidInstance
}
var (
err error
lst = m.getPartList()
)
if !abort {
if err = m.CheckSend(true, true); err != nil {
return err
}
}
if abort || len(lst) < 1 {
err = m.abortMPU()
} else {
err = m.completeMPU()
}
if !abort && err == nil && len(lst) < 1 && m.CurrentSizePart() > 0 {
err = m.SendObject()
}
m.callFuncOnComplete(abort, len(lst), m.getObject(), err)
if err != nil {
_ = m.closeWorkingFile()
}
m.cleanMPU()
return nil
}
func (m *mpu) cleanMPU() {
if m == nil {
return
}
m.m.Lock()
defer m.m.Unlock()
m.i = ""
m.o = ""
m.b = ""
m.l = nil
m.n = 0
}
func (m *mpu) SendObject() error {
var (
err error
cli *sdksss.Client
res *sdksss.PutObjectOutput
tmp libiot.FileProgress
ctx = m.getContext()
obj = m.getObject()
bck = m.getBucket()
tpe = m.getMimeType()
)
if cli = m.getClient(); cli == nil {
return ErrInvalidClient
} else if m.CurrentSizePart() < 1 {
return nil
} else if tmp, err = m.getWorkingFile(); err != nil {
return err
} else if tmp == nil {
return ErrInvalidTMPFile
} else if _, err = tmp.Seek(0, io.SeekStart); err != nil {
return err
}
res, err = cli.PutObject(ctx, &sdksss.PutObjectInput{
Bucket: sdkaws.String(bck),
Key: sdkaws.String(obj),
Body: tmp,
ContentType: sdkaws.String(tpe),
})
if err == nil {
if res == nil {
err = ErrInvalidResponse
} else if res.ETag == nil || len(*res.ETag) < 1 {
err = ErrInvalidResponse
}
}
return err
}
func (m *mpu) abortMPU() error {
var (
cli *sdksss.Client
err error
ctx = m.getContext()
obj = m.getObject()
bck = m.getBucket()
mid = m.getMultipartID()
mod = &sdksss.AbortMultipartUploadInput{
Bucket: sdkaws.String(bck),
Key: sdkaws.String(obj),
UploadId: sdkaws.String(mid),
}
)
if cli = m.getClient(); cli == nil {
return ErrInvalidClient
} else if len(mid) < 1 {
return nil
} else if _, err = cli.AbortMultipartUpload(ctx, mod); err != nil {
return err
}
return nil
}
func (m *mpu) completeMPU() error {
var (
cli *sdksss.Client
res *sdksss.CompleteMultipartUploadOutput
err error
ctx = m.getContext()
obj = m.getObject()
bck = m.getBucket()
mid = m.getMultipartID()
lst = m.getPartList()
mod = &sdksss.CompleteMultipartUploadInput{
Bucket: sdkaws.String(bck),
Key: sdkaws.String(obj),
UploadId: sdkaws.String(mid),
MultipartUpload: &sdktyp.CompletedMultipartUpload{
Parts: lst,
},
RequestPayer: sdktyp.RequestPayerRequester,
}
)
if cli = m.getClient(); cli == nil {
return ErrInvalidClient
} else if len(mid) < 1 {
return ErrInvalidUploadID
} else if res, err = cli.CompleteMultipartUpload(ctx, mod); err != nil {
return err
} else if res.Key == nil || len(*res.Key) < 1 {
return ErrInvalidResponse
}
return nil
}

87
aws/multipart/trigger.go Normal file
View File

@@ -0,0 +1,87 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package multipart
func (m *mpu) RegisterFuncOnPushPart(fct func(eTag string, e error)) {
if m == nil {
return
}
m.m.Lock()
defer m.m.Unlock()
m.fp = fct
}
func (m *mpu) callFuncOnPushPart(eTag string, e error) {
if m == nil {
return
}
m.m.RLock()
defer m.m.RUnlock()
if m.fp != nil {
m.fp(eTag, e)
}
}
func (m *mpu) RegisterFuncOnAbort(fct func(nPart int, obj string, e error)) {
if m == nil {
return
}
m.m.Lock()
defer m.m.Unlock()
m.fa = fct
}
func (m *mpu) RegisterFuncOnComplete(fct func(nPart int, obj string, e error)) {
if m == nil {
return
}
m.m.Lock()
defer m.m.Unlock()
m.fc = fct
}
func (m *mpu) callFuncOnComplete(abort bool, nPart int, obj string, e error) {
if m == nil {
return
}
m.m.RLock()
defer m.m.RUnlock()
if !abort && m.fc != nil {
m.fc(nPart, obj, e)
} else if abort && m.fa != nil {
m.fa(nPart, obj, e)
}
}

3
aws/object/interface.go
View File

@@ -30,6 +30,8 @@ import (
"io" "io"
"time" "time"
libmpu "github.com/nabbar/golib/aws/multipart"
libsiz "github.com/nabbar/golib/size" libsiz "github.com/nabbar/golib/size"
sdkiam "github.com/aws/aws-sdk-go-v2/service/iam" sdkiam "github.com/aws/aws-sdk-go-v2/service/iam"
@@ -67,6 +69,7 @@ type Object interface {
GetAttributes(object, version string) (*sdksss.GetObjectAttributesOutput, liberr.Error) GetAttributes(object, version string) (*sdksss.GetObjectAttributesOutput, liberr.Error)
MultipartList(keyMarker, markerId string) (uploads []sdktps.MultipartUpload, nextKeyMarker string, nextIdMarker string, count int64, e liberr.Error) MultipartList(keyMarker, markerId string) (uploads []sdktps.MultipartUpload, nextKeyMarker string, nextIdMarker string, count int64, e liberr.Error)
MultipartNew(partSize libsiz.Size, object string) libmpu.MultiPart
MultipartPut(object string, body io.Reader) liberr.Error MultipartPut(object string, body io.Reader) liberr.Error
MultipartPutCustom(partSize libsiz.Size, object string, body io.Reader) liberr.Error MultipartPutCustom(partSize libsiz.Size, object string, body io.Reader) liberr.Error
MultipartCancel(uploadId, key string) liberr.Error MultipartCancel(uploadId, key string) liberr.Error

158
aws/object/multipart.go
View File

@@ -26,29 +26,17 @@
package object package object
import ( import (
//nolint #nosec
/* #nosec */
"crypto/md5"
"encoding/base64"
"io" "io"
"mime"
"path/filepath"
libsiz "github.com/nabbar/golib/size"
//nolint #gci
"os"
sdkaws "github.com/aws/aws-sdk-go-v2/aws" sdkaws "github.com/aws/aws-sdk-go-v2/aws"
sdksss "github.com/aws/aws-sdk-go-v2/service/s3" sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
sdktyp "github.com/aws/aws-sdk-go-v2/service/s3/types" sdktyp "github.com/aws/aws-sdk-go-v2/service/s3/types"
libhlp "github.com/nabbar/golib/aws/helper" libhlp "github.com/nabbar/golib/aws/helper"
libmpu "github.com/nabbar/golib/aws/multipart"
liberr "github.com/nabbar/golib/errors" liberr "github.com/nabbar/golib/errors"
libiou "github.com/nabbar/golib/ioutils" libsiz "github.com/nabbar/golib/size"
) )
const DefaultPartSize = 5 * libsiz.SizeMega
// MultipartList implement the ListMultipartUploads. // MultipartList implement the ListMultipartUploads.
// See docs for more infos : https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html // See docs for more infos : https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html
func (cli *client) MultipartList(keyMarker, markerId string) (uploads []sdktyp.MultipartUpload, nextKeyMarker string, nextIdMarker string, count int64, e liberr.Error) { func (cli *client) MultipartList(keyMarker, markerId string) (uploads []sdktyp.MultipartUpload, nextKeyMarker string, nextIdMarker string, count int64, e liberr.Error) {
@@ -73,141 +61,57 @@ func (cli *client) MultipartList(keyMarker, markerId string) (uploads []sdktyp.M
} }
} }
func (cli *client) MultipartNew(partSize libsiz.Size, object string) libmpu.MultiPart {
m := libmpu.New(partSize, object, cli.GetBucketName())
m.RegisterContext(cli.GetContext)
m.RegisterClientS3(func() *sdksss.Client {
return cli.s3
})
return m
}
func (cli *client) MultipartPut(object string, body io.Reader) liberr.Error { func (cli *client) MultipartPut(object string, body io.Reader) liberr.Error {
return cli.MultipartPutCustom(DefaultPartSize, object, body) return cli.MultipartPutCustom(libmpu.DefaultPartSize, object, body)
} }
func (cli *client) MultipartPutCustom(partSize libsiz.Size, object string, body io.Reader) liberr.Error { func (cli *client) MultipartPutCustom(partSize libsiz.Size, object string, body io.Reader) liberr.Error {
var ( var (
tmp libiou.FileProgress e error
rio libhlp.ReaderPartSize m = cli.MultipartNew(partSize, object)
upl *sdksss.CreateMultipartUploadOutput
err error
tpe *string
) )
defer func() { defer func() {
if tmp != nil { if m != nil {
_ = tmp.Close() _ = m.Close()
} }
}() }()
if t := mime.TypeByExtension(filepath.Ext(object)); t == "" { if e = m.StartMPU(); e != nil {
tpe = sdkaws.String("application/octet-stream") return cli.GetError(e)
} else if _, e = io.Copy(m, body); e != nil {
return cli.GetError(e)
} else if e = m.StopMPU(false); e != nil {
return cli.GetError(e)
} else { } else {
tpe = sdkaws.String(t) m = nil
}
upl, err = cli.s3.CreateMultipartUpload(cli.GetContext(), &sdksss.CreateMultipartUploadInput{
Key: sdkaws.String(object),
Bucket: sdkaws.String(cli.GetBucketName()),
ContentType: tpe,
})
if err != nil {
return cli.GetError(err)
} else if upl == nil {
return libhlp.ErrorResponse.Error(nil)
}
rio = libhlp.NewReaderPartSize(body, partSize)
for !rio.IeOEF() {
var (
inf os.FileInfo
prt *sdksss.UploadPartOutput
)
tmp, err = libiou.NewFileProgressTemp()
if err != nil {
return cli._MultipartCancel(err, upl.UploadId, object)
}
_, err = io.Copy(tmp, rio)
if err != nil {
return cli._MultipartCancel(err, upl.UploadId, object)
}
_, err = tmp.Seek(0, io.SeekStart)
if err != nil {
return cli._MultipartCancel(err, upl.UploadId, object)
}
inf, err = tmp.FileStat()
if err != nil {
return cli._MultipartCancel(err, upl.UploadId, object)
}
/* #nosec */
h := md5.New()
if _, e := tmp.WriteTo(h); e != nil {
return cli._MultipartCancel(e, upl.UploadId, object)
}
_, err = tmp.Seek(0, io.SeekStart)
if err != nil {
return cli._MultipartCancel(err, upl.UploadId, object)
}
prt, err = cli.s3.UploadPart(cli.GetContext(), &sdksss.UploadPartInput{
Bucket: sdkaws.String(cli.GetBucketName()),
Body: tmp,
PartNumber: rio.CurrPart(),
UploadId: upl.UploadId,
Key: sdkaws.String(object),
ContentLength: inf.Size(),
RequestPayer: sdktyp.RequestPayerRequester,
ContentMD5: sdkaws.String(base64.StdEncoding.EncodeToString(h.Sum(nil))),
})
_ = tmp.Close()
tmp = nil
if err != nil {
return cli._MultipartCancel(err, upl.UploadId, object)
} else if prt == nil || prt.ETag == nil || len(*prt.ETag) == 0 {
return cli._MultipartCancel(libhlp.ErrorResponse.Error(nil), upl.UploadId, object)
}
rio.NextPart(prt.ETag)
}
var prt *sdksss.CompleteMultipartUploadOutput
prt, err = cli.s3.CompleteMultipartUpload(cli.GetContext(), &sdksss.CompleteMultipartUploadInput{
Bucket: sdkaws.String(cli.GetBucketName()),
Key: sdkaws.String(object),
UploadId: upl.UploadId,
MultipartUpload: rio.CompPart(),
RequestPayer: sdktyp.RequestPayerRequester,
})
if err != nil {
return cli._MultipartCancel(err, upl.UploadId, object)
} else if prt == nil || prt.ETag == nil || len(*prt.ETag) == 0 {
return cli._MultipartCancel(libhlp.ErrorResponse.Error(nil), upl.UploadId, object)
} }
return nil return nil
} }
func (cli *client) _MultipartCancel(err error, updIp *string, object string) liberr.Error { func (cli *client) MultipartCancel(uploadId, key string) liberr.Error {
cnl, e := cli.s3.AbortMultipartUpload(cli.GetContext(), &sdksss.AbortMultipartUploadInput{ res, err := cli.s3.AbortMultipartUpload(cli.GetContext(), &sdksss.AbortMultipartUploadInput{
Bucket: sdkaws.String(cli.GetBucketName()), Bucket: sdkaws.String(cli.GetBucketName()),
UploadId: updIp, UploadId: sdkaws.String(uploadId),
Key: sdkaws.String(object), Key: sdkaws.String(key),
}) })
if e != nil { if err != nil {
return cli.GetError(e, err)
} else if cnl == nil {
return libhlp.ErrorResponse.Error(cli.GetError(err))
} else if err != nil {
return cli.GetError(err) return cli.GetError(err)
} else if res == nil {
return libhlp.ErrorResponse.Error(nil)
} else { } else {
return nil return nil
} }
} }
func (cli *client) MultipartCancel(uploadId, key string) liberr.Error {
return cli._MultipartCancel(nil, sdkaws.String(uploadId), key)
}

4
aws/object_test.go
View File

@@ -28,6 +28,8 @@ package aws_test
import ( import (
"bytes" "bytes"
libsiz "github.com/nabbar/golib/size"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
) )
@@ -70,7 +72,7 @@ var _ = Describe("Object", func() {
Context("Multipart Put object", func() { Context("Multipart Put object", func() {
It("Must fail as the bucket doesn't exists - 5", func() { It("Must fail as the bucket doesn't exists - 5", func() {
err := cli.Object().MultipartPut("object", randContent(4*1024)) err := cli.Object().MultipartPut("object", randContent(10*libsiz.SizeMega))
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
}) })
}) })

15
cobra/configure.go
View File

@@ -79,7 +79,7 @@ func (c *cobra) ConfigureCheckArgs(basename string, args []string) error {
return nil return nil
} }
func (c *cobra) ConfigureWriteConfig(basename string, defaultConfig func() io.Reader) error { func (c *cobra) ConfigureWriteConfig(basename string, defaultConfig func() io.Reader, printMsg func(pkg, file string)) error {
pkg := c.getPackageName() pkg := c.getPackageName()
if basename == "" && pkg != "" { if basename == "" && pkg != "" {
@@ -137,9 +137,14 @@ func (c *cobra) ConfigureWriteConfig(basename string, defaultConfig func() io.Re
return err return err
} }
println(fmt.Sprintf("\n\t>> Config File '%s' has been created and file permission have been set.", cfgFile)) if printMsg == nil {
println("\t>> To explicitly specify this config file when you call this tool, use the '-c' flag like this: ") println(fmt.Sprintf("\n\t>> Config File '%s' has been created and file permission have been set.", cfgFile))
println(fmt.Sprintf("\t\t\t %s -c %s <cmd>...\n", pkg, cfgFile)) println("\t>> To explicitly specify this config file when you call this tool, use the '-c' flag like this: ")
println(fmt.Sprintf("\t\t\t %s -c %s <cmd>...\n", pkg, cfgFile))
} else {
printMsg(pkg, cfgFile)
}
return nil return nil
} }
@@ -198,7 +203,7 @@ func (c *cobra) AddCommandConfigure(basename string, defaultConfig func() io.Rea
override by passed flag in command line and completed with default for non existing values.`, override by passed flag in command line and completed with default for non existing values.`,
RunE: func(cmd *spfcbr.Command, args []string) error { RunE: func(cmd *spfcbr.Command, args []string) error {
return c.ConfigureWriteConfig(basename, defaultConfig) return c.ConfigureWriteConfig(basename, defaultConfig, nil)
}, },
Args: func(cmd *spfcbr.Command, args []string) error { Args: func(cmd *spfcbr.Command, args []string) error {

2
cobra/interface.go
View File

@@ -65,7 +65,7 @@ type Cobra interface {
Cobra() *spfcbr.Command Cobra() *spfcbr.Command
ConfigureCheckArgs(basename string, args []string) error ConfigureCheckArgs(basename string, args []string) error
ConfigureWriteConfig(basename string, defaultConfig func() io.Reader) error ConfigureWriteConfig(basename string, defaultConfig func() io.Reader, printMsg func(pkg, file string)) error
} }
func New() Cobra { func New() Cobra {

4
console/color.go
View File

@@ -26,8 +26,8 @@
package console package console
import ( import (
"bufio"
"fmt" "fmt"
"io"
"github.com/fatih/color" "github.com/fatih/color"
"github.com/nabbar/golib/errors" "github.com/nabbar/golib/errors"
@@ -89,7 +89,7 @@ func (c colorType) Print(text string) {
} }
} }
func (c colorType) BuffPrintf(buff *bufio.ReadWriter, format string, args ...interface{}) (n int, err errors.Error) { func (c colorType) BuffPrintf(buff io.Writer, format string, args ...interface{}) (n int, err errors.Error) {
if colorList[c] != nil && buff != nil { if colorList[c] != nil && buff != nil {
//nolint #nosec //nolint #nosec

65
go.mod
View File

@@ -13,39 +13,39 @@ require (
github.com/fatih/color v1.15.0 github.com/fatih/color v1.15.0
github.com/fsnotify/fsnotify v1.6.0 github.com/fsnotify/fsnotify v1.6.0
github.com/fxamacker/cbor/v2 v2.4.0 github.com/fxamacker/cbor/v2 v2.4.0
github.com/gin-gonic/gin v1.9.0 github.com/gin-gonic/gin v1.9.1
github.com/go-ldap/ldap/v3 v3.4.4 github.com/go-ldap/ldap/v3 v3.4.4
github.com/go-playground/validator/v10 v10.13.0 github.com/go-playground/validator/v10 v10.14.0
github.com/google/go-github/v33 v33.0.0 github.com/google/go-github/v33 v33.0.0
github.com/hashicorp/go-hclog v1.5.0 github.com/hashicorp/go-hclog v1.5.0
github.com/hashicorp/go-retryablehttp v0.7.2 github.com/hashicorp/go-retryablehttp v0.7.2
github.com/hashicorp/go-uuid v1.0.3 github.com/hashicorp/go-uuid v1.0.3
github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/go-version v1.6.0
github.com/jlaffaye/ftp v0.1.0 github.com/jlaffaye/ftp v0.2.0
github.com/lni/dragonboat/v3 v3.3.6 github.com/lni/dragonboat/v3 v3.3.6
github.com/matcornic/hermes/v2 v2.1.0 github.com/matcornic/hermes/v2 v2.1.0
github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-colorable v0.1.13
github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/mapstructure v1.5.0 github.com/mitchellh/mapstructure v1.5.0
github.com/nats-io/jwt/v2 v2.4.1 github.com/nats-io/jwt/v2 v2.4.1
github.com/nats-io/nats-server/v2 v2.9.16 github.com/nats-io/nats-server/v2 v2.9.17
github.com/nats-io/nats.go v1.25.0 github.com/nats-io/nats.go v1.26.0
github.com/nutsdb/nutsdb v0.12.0 github.com/nutsdb/nutsdb v0.12.2
github.com/onsi/ginkgo/v2 v2.9.4 github.com/onsi/ginkgo/v2 v2.9.7
github.com/onsi/gomega v1.27.6 github.com/onsi/gomega v1.27.7
github.com/pelletier/go-toml v1.9.5 github.com/pelletier/go-toml v1.9.5
github.com/prometheus/client_golang v1.15.1 github.com/prometheus/client_golang v1.15.1
github.com/shirou/gopsutil v3.21.11+incompatible github.com/shirou/gopsutil v3.21.11+incompatible
github.com/sirupsen/logrus v1.9.0 github.com/sirupsen/logrus v1.9.2
github.com/spf13/cobra v1.7.0 github.com/spf13/cobra v1.7.0
github.com/spf13/jwalterweatherman v1.1.0 github.com/spf13/jwalterweatherman v1.1.0
github.com/spf13/viper v1.15.0 github.com/spf13/viper v1.16.0
github.com/ugorji/go/codec v1.2.11 github.com/ugorji/go/codec v1.2.11
github.com/vbauerster/mpb/v5 v5.4.0 github.com/vbauerster/mpb/v5 v5.4.0
github.com/xanzy/go-gitlab v0.83.0 github.com/xanzy/go-gitlab v0.84.0
github.com/xhit/go-simple-mail v2.2.2+incompatible github.com/xhit/go-simple-mail v2.2.2+incompatible
github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235
golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
golang.org/x/net v0.10.0 golang.org/x/net v0.10.0
golang.org/x/oauth2 v0.8.0 golang.org/x/oauth2 v0.8.0
golang.org/x/sync v0.2.0 golang.org/x/sync v0.2.0
@@ -53,22 +53,22 @@ require (
golang.org/x/term v0.8.0 golang.org/x/term v0.8.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/clickhouse v0.5.1 gorm.io/driver/clickhouse v0.5.1
gorm.io/driver/mysql v1.5.0 gorm.io/driver/mysql v1.5.1
gorm.io/driver/postgres v1.5.0 gorm.io/driver/postgres v1.5.2
gorm.io/driver/sqlite v1.5.0 gorm.io/driver/sqlite v1.5.1
gorm.io/driver/sqlserver v1.4.3 gorm.io/driver/sqlserver v1.5.0
gorm.io/gorm v1.25.1 gorm.io/gorm v1.25.1
) )
require ( require (
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/ClickHouse/ch-go v0.55.0 // indirect github.com/ClickHouse/ch-go v0.56.0 // indirect
github.com/ClickHouse/clickhouse-go/v2 v2.9.3 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.10.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/Masterminds/sprig v2.22.0+incompatible // indirect
github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/PuerkitoBio/goquery v1.8.1 // indirect
github.com/VictoriaMetrics/metrics v1.23.1 // indirect github.com/VictoriaMetrics/metrics v1.24.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/andybalholm/brotli v1.0.5 // indirect github.com/andybalholm/brotli v1.0.5 // indirect
@@ -90,13 +90,14 @@ require (
github.com/aws/smithy-go v1.13.5 // indirect github.com/aws/smithy-go v1.13.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/bwmarrin/snowflake v0.3.0 // indirect github.com/bwmarrin/snowflake v0.3.0 // indirect
github.com/bytedance/sonic v1.8.8 // indirect github.com/bytedance/sonic v1.9.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/errors v1.9.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb // indirect github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb // indirect
github.com/cockroachdb/redact v1.1.4 // indirect github.com/cockroachdb/redact v1.1.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/getsentry/sentry-go v0.21.0 // indirect github.com/getsentry/sentry-go v0.21.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
@@ -117,7 +118,7 @@ require (
github.com/google/btree v1.1.2 // indirect github.com/google/btree v1.1.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-cmp v0.5.9 // indirect
github.com/google/go-querystring v1.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 // indirect github.com/google/pprof v0.0.0-20230602010524-ada837c32108 // indirect
github.com/google/uuid v1.3.0 // indirect github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/css v1.0.0 // indirect github.com/gorilla/css v1.0.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -130,7 +131,7 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/memberlist v0.5.0 // indirect github.com/hashicorp/memberlist v0.5.0 // indirect
github.com/huandu/xstrings v1.4.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.15 // indirect github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
@@ -147,12 +148,12 @@ require (
github.com/leodido/go-urn v1.2.4 // indirect github.com/leodido/go-urn v1.2.4 // indirect
github.com/lni/goutils v1.3.0 // indirect github.com/lni/goutils v1.3.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-isatty v0.0.18 // indirect github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-sqlite3 v1.14.16 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect
github.com/mattn/go-tty v0.0.5 // indirect github.com/mattn/go-tty v0.0.5 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/microsoft/go-mssqldb v0.21.0 // indirect github.com/microsoft/go-mssqldb v1.1.0 // indirect
github.com/miekg/dns v1.1.54 // indirect github.com/miekg/dns v1.1.54 // indirect
github.com/minio/highwayhash v1.0.2 // indirect github.com/minio/highwayhash v1.0.2 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect
@@ -163,13 +164,13 @@ require (
github.com/nats-io/nuid v1.0.1 // indirect github.com/nats-io/nuid v1.0.1 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/paulmach/orb v0.9.2 // indirect github.com/paulmach/orb v0.9.2 // indirect
github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pkg/errors v0.9.1 // indirect github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/term v1.2.0-beta.2 // indirect github.com/pkg/term v1.2.0-beta.2 // indirect
github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.43.0 // indirect github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect
github.com/rivo/uniseg v0.4.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect
@@ -177,7 +178,7 @@ require (
github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/asm v1.2.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect github.com/shopspring/decimal v1.3.1 // indirect
github.com/spf13/afero v1.9.5 // indirect github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
github.com/subosito/gotenv v1.4.2 // indirect github.com/subosito/gotenv v1.4.2 // indirect
@@ -189,14 +190,14 @@ require (
github.com/x448/float16 v0.8.4 // indirect github.com/x448/float16 v0.8.4 // indirect
github.com/xujiajun/mmap-go v1.0.1 // indirect github.com/xujiajun/mmap-go v1.0.1 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opentelemetry.io/otel v1.15.1 // indirect go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/trace v1.15.1 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect
golang.org/x/arch v0.3.0 // indirect golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.9.0 // indirect golang.org/x/crypto v0.9.0 // indirect
golang.org/x/mod v0.10.0 // indirect golang.org/x/mod v0.10.0 // indirect
golang.org/x/text v0.9.0 // indirect golang.org/x/text v0.9.0 // indirect
golang.org/x/time v0.3.0 // indirect golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.9.1 // indirect golang.org/x/tools v0.9.3 // indirect
google.golang.org/appengine v1.6.7 // indirect google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.30.0 // indirect google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect

9
httpcli/network.go
View File

@@ -33,12 +33,15 @@ type Network uint8
const ( const (
NetworkTCP Network = iota NetworkTCP Network = iota
NetworkUDP NetworkUDP
NetworkUnix
) )
func GetNetworkFromString(str string) Network { func GetNetworkFromString(str string) Network {
switch strings.ToLower(str) { switch {
case NetworkUDP.Code(): case strings.EqualFold(NetworkUDP.Code(), str):
return NetworkUDP return NetworkUDP
case strings.EqualFold(NetworkUnix.Code(), str):
return NetworkUnix
default: default:
return NetworkTCP return NetworkTCP
} }
@@ -48,6 +51,8 @@ func (n Network) String() string {
switch n { switch n {
case NetworkUDP: case NetworkUDP:
return "UDP" return "UDP"
case NetworkUnix:
return "unix"
default: default:
return "TCP" return "TCP"
} }

6
ioutils/error.go
View File

@@ -38,6 +38,8 @@ const (
ErrorSyscallRLimitSet ErrorSyscallRLimitSet
ErrorIOFileStat ErrorIOFileStat
ErrorIOFileSeek ErrorIOFileSeek
ErrorIOFileTruncate
ErrorIOFileSync
ErrorIOFileOpen ErrorIOFileOpen
ErrorIOFileTempNew ErrorIOFileTempNew
ErrorIOFileTempClose ErrorIOFileTempClose
@@ -64,6 +66,10 @@ func getMessage(code liberr.CodeError) (message string) {
return "error occur while trying to get stat of file" return "error occur while trying to get stat of file"
case ErrorIOFileSeek: case ErrorIOFileSeek:
return "error occur while trying seek into file" return "error occur while trying seek into file"
case ErrorIOFileTruncate:
return "error occur while trying truncate file"
case ErrorIOFileSync:
return "error occur while trying to sync file"
case ErrorIOFileOpen: case ErrorIOFileOpen:
return "error occur while trying to open file" return "error occur while trying to open file"
case ErrorIOFileTempNew: case ErrorIOFileTempNew:

34
ioutils/fileProgess.go
View File

@@ -67,6 +67,8 @@ type FileProgress interface {
FileStat() (os.FileInfo, liberr.Error) FileStat() (os.FileInfo, liberr.Error)
SizeToEOF() (size int64, err liberr.Error) SizeToEOF() (size int64, err liberr.Error)
Truncate(size int64) liberr.Error
Sync() liberr.Error
NewFilePathMode(filepath string, mode int, perm os.FileMode) (FileProgress, liberr.Error) NewFilePathMode(filepath string, mode int, perm os.FileMode) (FileProgress, liberr.Error)
NewFilePathWrite(filepath string, create, overwrite bool, perm os.FileMode) (FileProgress, liberr.Error) NewFilePathWrite(filepath string, create, overwrite bool, perm os.FileMode) (FileProgress, liberr.Error)
@@ -123,12 +125,14 @@ func NewFileProgressPathMode(filepath string, mode int, perm os.FileMode) (FileP
} }
func NewFileProgressPathWrite(filepath string, create, overwrite bool, perm os.FileMode) (FileProgress, liberr.Error) { func NewFileProgressPathWrite(filepath string, create, overwrite bool, perm os.FileMode) (FileProgress, liberr.Error) {
mode := os.O_RDWR | os.O_TRUNC var mode = os.O_RDWR
if _, err := os.Stat(filepath); err != nil && os.IsNotExist(err) && create { if _, err := os.Stat(filepath); err != nil && os.IsNotExist(err) && create {
mode = os.O_RDWR | os.O_CREATE | os.O_TRUNC mode = os.O_RDWR | os.O_CREATE
} else if err != nil { } else if err != nil {
return nil, ErrorIOFileStat.ErrorParent(err) return nil, ErrorIOFileStat.ErrorParent(err)
} else if err == nil && overwrite {
mode = os.O_RDWR | os.O_TRUNC
} }
return NewFileProgressPathMode(filepath, mode, perm) return NewFileProgressPathMode(filepath, mode, perm)
@@ -271,6 +275,32 @@ func (f *fileProgress) SizeToEOF() (size int64, err liberr.Error) {
} }
} }
func (f *fileProgress) Truncate(size int64) liberr.Error {
if f == nil {
return ErrorNilPointer.Error(nil)
}
if e := f.fs.Truncate(size); e != nil {
return ErrorIOFileTruncate.ErrorParent(e)
}
f.reset(0)
return nil
}
func (f *fileProgress) Sync() liberr.Error {
if f == nil {
return ErrorNilPointer.Error(nil)
}
if e := f.fs.Sync(); e != nil {
return ErrorIOFileSync.ErrorParent(e)
}
return nil
}
func (f *fileProgress) SetIncrement(increment func(size int64)) { func (f *fileProgress) SetIncrement(increment func(size int64)) {
if f != nil { if f != nil {
f.fc = increment f.fc = increment