Package Archive: add Helper & Compress DetectOnly (#199)

Package archive/helper
- adding package to compress/uncompress with reader or writer
- refactor to allowing to use same source of io as result: io.reader or
io.writer
- optimize code & buf to limit mem use
- rework variable to be thread safe

Package archive/compress
- add function DetectOnly to detect algo and return an updated reader
but not the decompressor reader
- update function Detect to use DetectOnly to limit duplication code

Other
- bump dependencies


## Type of Change
Please select the type of change your PR introduces by checking the
appropriate box:
- [ ] Fixes an issue
- [X] Adds a new feature
- [X] Refactor
- [ ] Documentation update
- [ ] Other (please describe it in the Description Section)

archive/compress/interface.go

@@ -40,12 +40,27 @@ func Parse(s string) Algorithm {
 }
 
 func Detect(r io.Reader) (Algorithm, io.ReadCloser, error) {
+	var (
+		err error
+		alg Algorithm
+		rdr io.ReadCloser
+	)
+
+	if alg, rdr, err = DetectOnly(r); err != nil {
+		return None, nil, err
+	} else if rdr, err = alg.Reader(rdr); err != nil {
+		return None, nil, err
+	} else {
+		return alg, rdr, nil
+	}
+}
+
+func DetectOnly(r io.Reader) (Algorithm, io.ReadCloser, error) {
 	var (
 		err error
 		alg Algorithm
 		bfr = bufio.NewReader(r)
 		buf []byte
-		res io.ReadCloser
 	)
 
 	if buf, err = bfr.Peek(6); err != nil {
@@ -66,11 +81,5 @@ func Detect(r io.Reader) (Algorithm, io.ReadCloser, error) {
 		alg = None
 	}
 
-	if err != nil {
-		return None, nil, err
-	} else if res, err = alg.Reader(bfr); err != nil {
-		return None, nil, err
-	} else {
-		return alg, res, err
-	}
+	return alg, io.NopCloser(bfr), err
 }

archive/compress/types.go

@@ -37,6 +37,27 @@ const (
 	XZ
 )
 
+func List() []Algorithm {
+	return []Algorithm{
+		None,
+		Bzip2,
+		Gzip,
+		LZ4,
+		XZ,
+	}
+}
+
+func ListString() []string {
+	var (
+		lst = List()
+		res = make([]string, len(lst))
+	)
+	for i := range lst {
+		res[i] = lst[i].String()
+	}
+	return res
+}
+
 func (a Algorithm) IsNone() bool {
 	return a == None
 }

archive/helper/compressor.go

@@ -0,0 +1,157 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2024 Salim Amine BOU ARAM & Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package helper
+
+import (
+	"bytes"
+	"io"
+	"sync/atomic"
+)
+
+type compressWriter struct {
+	dst io.WriteCloser
+}
+
+func (o *compressWriter) Read(p []byte) (n int, err error) {
+	return 0, ErrInvalidSource
+}
+
+func (o *compressWriter) Write(p []byte) (n int, err error) {
+	return o.dst.Write(p)
+}
+
+func (o *compressWriter) Close() error {
+	return o.dst.Close()
+}
+
+// compressor handles data compression in chunks.
+type compressReader struct {
+	src io.ReadCloser
+	wrt io.WriteCloser
+	buf *bytes.Buffer
+	clo *atomic.Bool
+}
+
+// Read for compressor compresses the data and reads it from the buffer in chunks.
+func (o *compressReader) Read(p []byte) (n int, err error) {
+	if o.src == nil {
+		return 0, ErrInvalidSource
+	}
+
+	var size int
+
+	if s := cap(p); s < chunkSize {
+		size = chunkSize
+	} else {
+		size = s
+	}
+
+	if o.clo.Load() && o.buf.Len() == 0 {
+		return 0, io.EOF
+	}
+
+	if o.buf.Len() < size && !o.clo.Load() {
+		if _, err = o.fill(size); err != nil {
+			return 0, err
+		}
+	}
+
+	n, err = o.buf.Read(p)
+
+	if n > 0 {
+		return n, nil
+	} else if err == nil {
+		err = io.EOF
+	}
+
+	return 0, err
+}
+
+// fill handles compressing data from the source and writing to the buffer.
+func (o *compressReader) fill(size int) (n int, err error) {
+	var (
+		buf    = make([]byte, size)
+		errWrt error
+		errclo error
+	)
+
+	for o.buf.Len() < size {
+		if n, err = o.src.Read(buf); err != nil && err != io.EOF {
+			return 0, err
+		}
+
+		if n > 0 {
+			if _, errWrt = o.wrt.Write(buf[:n]); errWrt != nil {
+				return 0, errWrt
+			}
+		}
+
+		if err == io.EOF {
+			o.clo.Store(true)
+
+			errWrt = o.wrt.Close()
+			errclo = o.src.Close()
+
+			if errclo != nil {
+				return 0, errclo
+			} else if errWrt != nil {
+				return 0, errWrt
+			}
+
+			return o.buf.Len(), nil
+		} else if err != nil {
+			return n, err
+		}
+	}
+
+	data := o.buf.Bytes()
+	o.buf.Reset()
+
+	if _, err = o.buf.Write(data); err != nil {
+		return 0, err
+	}
+
+	return o.buf.Len(), nil
+}
+
+// Close closes the compressor and underlying writer.
+func (o *compressReader) Close() (err error) {
+	a := o.clo.Swap(true)
+
+	if o.buf != nil {
+		o.buf.Reset()
+	}
+
+	if o.wrt != nil && !a {
+		return o.wrt.Close()
+	}
+
+	return nil
+}
+
+func (o *compressReader) Write(p []byte) (n int, err error) {
+	return 0, ErrInvalidSource
+}

archive/helper/decompressor.go

@@ -0,0 +1,180 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2024 Salim Amine BOU ARAM & Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package helper
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	arccmp "github.com/nabbar/golib/archive/compress"
+)
+
+const workBufSizeDeCompressWrite = 32 * 1024 // 32kB for buffer
+
+type deCompressReader struct {
+	src io.ReadCloser
+}
+
+func (o *deCompressReader) Read(p []byte) (n int, err error) {
+	return o.src.Read(p)
+}
+
+func (o *deCompressReader) Write(p []byte) (n int, err error) {
+	return 0, ErrInvalidSource
+}
+
+func (o *deCompressReader) Close() error {
+	return o.src.Close()
+}
+
+type bufNoEOF struct {
+	m sync.Mutex
+	b *bytes.Buffer
+	c *atomic.Bool
+}
+
+func (o *bufNoEOF) Read(p []byte) (n int, err error) {
+	if o.c.Load() && o.b.Len() < 1 {
+		return 0, io.EOF
+	}
+
+	for o.b.Len() < 1 && !o.c.Load() {
+		time.Sleep(100 * time.Microsecond)
+	}
+
+	n, _ = o.readBuff(p)
+
+	if n < 1 && o.c.Load() {
+		return 0, io.EOF
+	} else {
+		return n, nil
+	}
+}
+
+func (o *bufNoEOF) Write(p []byte) (n int, err error) {
+	if o.c.Load() {
+		return 0, errors.New("closed buffer")
+	}
+
+	return o.writeBuff(p)
+}
+
+func (o *bufNoEOF) Close() error {
+	o.c.Store(true)
+	return nil
+}
+
+func (o *bufNoEOF) Len() int {
+	o.m.Lock()
+	defer o.m.Unlock()
+	return o.b.Len()
+}
+
+func (o *bufNoEOF) readBuff(p []byte) (n int, err error) {
+	o.m.Lock()
+	defer o.m.Unlock()
+
+	if o.b.Len() < 1 {
+		return 0, io.EOF
+	}
+
+	buf := bytes.NewBuffer(make([]byte, 0))
+	buf.Write(o.b.Bytes())
+
+	n, err = buf.Read(p)
+	o.b.Reset()
+
+	var e error
+	if buf.Len() > 0 {
+		_, e = io.Copy(o.b, buf)
+	}
+
+	if err == nil && e != nil && e != io.EOF {
+		return n, e
+	}
+
+	return n, err
+}
+
+func (o *bufNoEOF) writeBuff(p []byte) (n int, err error) {
+	o.m.Lock()
+	defer o.m.Unlock()
+	return o.b.Write(p)
+}
+
+type deCompressWriter struct {
+	alg arccmp.Algorithm
+	wrt io.WriteCloser
+	buf *bufNoEOF
+	clo *atomic.Bool
+	run *atomic.Bool
+}
+
+func (o *deCompressWriter) Read(p []byte) (n int, err error) {
+	return 0, ErrInvalidSource
+}
+
+func (o *deCompressWriter) Write(p []byte) (n int, err error) {
+	if o.clo.Load() {
+		return 0, ErrClosedResource
+	}
+
+	n, err = o.buf.Write(p)
+	if err != nil || o.run.Load() {
+		return n, err
+	}
+
+	o.run.Store(true)
+	if r, e := o.alg.Reader(o.buf); e != nil {
+		return n, e
+	} else {
+		go func() {
+			_, _ = io.Copy(o.wrt, r)
+		}()
+	}
+
+	return n, err
+}
+
+func (o *deCompressWriter) Close() error {
+	o.clo.Store(true)
+	o.run.Store(false)
+	_ = o.buf.Close()
+
+	for o.buf.Len() > 0 {
+		time.Sleep(time.Millisecond)
+	}
+
+	if err := o.wrt.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}

archive/helper/interface.go

@@ -0,0 +1,155 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2024 Salim Amine BOU ARAM & Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package helper
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"sync"
+	"sync/atomic"
+
+	libarc "github.com/nabbar/golib/archive"
+	arccmp "github.com/nabbar/golib/archive/compress"
+)
+
+const chunkSize = 512
+
+var (
+	ErrInvalidSource    = errors.New("invalid source")
+	ErrClosedResource   = errors.New("closed resource")
+	ErrInvalidOperation = errors.New("invalid operation")
+)
+
+type Helper interface {
+	io.ReadWriteCloser
+}
+
+func New(algo arccmp.Algorithm, ope Operation, src any) (h Helper, err error) {
+	if r, k := src.(io.Reader); k {
+		return NewReader(algo, ope, r)
+	}
+	if w, k := src.(io.Writer); k {
+		return NewWriter(algo, ope, w)
+	}
+	return nil, ErrInvalidSource
+}
+
+func NewReader(algo arccmp.Algorithm, ope Operation, src io.Reader) (Helper, error) {
+	switch ope {
+	case Compress:
+		return makeCompressReader(algo, src)
+	case Decompress:
+		return makeDeCompressReader(algo, src)
+	}
+
+	return nil, ErrInvalidOperation
+}
+
+func NewWriter(algo arccmp.Algorithm, ope Operation, dst io.Writer) (Helper, error) {
+	switch ope {
+	case Compress:
+		return makeCompressWriter(algo, dst)
+	case Decompress:
+		return makeDeCompressWriter(algo, dst)
+	}
+
+	return nil, ErrInvalidOperation
+}
+
+func makeCompressWriter(algo arccmp.Algorithm, src io.Writer) (h Helper, err error) {
+	wc, ok := src.(io.WriteCloser)
+
+	if !ok {
+		wc = libarc.NopWriteCloser(src)
+	}
+
+	if wc, err = algo.Writer(wc); err != nil {
+		return nil, err
+	} else {
+		return &compressWriter{
+			dst: wc,
+		}, nil
+	}
+}
+
+func makeCompressReader(algo arccmp.Algorithm, src io.Reader) (h Helper, err error) {
+	rc, ok := src.(io.ReadCloser)
+
+	if !ok {
+		rc = io.NopCloser(src)
+	}
+
+	var (
+		buf = bytes.NewBuffer(make([]byte, 0))
+		wrt io.WriteCloser
+	)
+
+	wrt, err = algo.Writer(libarc.NopWriteCloser(buf))
+
+	return &compressReader{
+		src: rc,
+		wrt: wrt,
+		buf: buf,
+		clo: new(atomic.Bool),
+	}, nil
+}
+
+func makeDeCompressReader(algo arccmp.Algorithm, src io.Reader) (h Helper, err error) {
+	rc, ok := src.(io.ReadCloser)
+
+	if !ok {
+		rc = io.NopCloser(src)
+	}
+
+	if rc, err = algo.Reader(rc); err != nil {
+		return nil, err
+	} else {
+		return &deCompressReader{
+			src: rc,
+		}, nil
+	}
+}
+
+func makeDeCompressWriter(algo arccmp.Algorithm, src io.Writer) (h Helper, err error) {
+	wc, ok := src.(io.WriteCloser)
+
+	if !ok {
+		wc = libarc.NopWriteCloser(src)
+	}
+
+	return &deCompressWriter{
+		alg: algo,
+		wrt: wc,
+		buf: &bufNoEOF{
+			m: sync.Mutex{},
+			b: bytes.NewBuffer(make([]byte, 0)),
+			c: new(atomic.Bool),
+		},
+		clo: new(atomic.Bool),
+		run: new(atomic.Bool),
+	}, nil
+}

archive/helper/types.go

@@ -0,0 +1,33 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2024 Salim Amine BOU ARAM & Nicolas JUHEL
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package helper
+
+type Operation uint8
+
+const (
+	Compress Operation = iota
+	Decompress
+)

archive/helper_compress_test.go

@@ -0,0 +1,120 @@
+/*
+ *  MIT License
+ *
+ *  Copyright (c) 2024 Salim Amine Bou Aram
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ */
+
+package archive_test
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"time"
+
+	arccmp "github.com/nabbar/golib/archive/compress"
+	archlp "github.com/nabbar/golib/archive/helper"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Compress Helper Test", func() {
+	for _, algo := range arccmp.List() {
+		Context("For the algo '"+algo.String()+"', in reader mode", func() {
+			It("should compress/decompress in embedded stream correctly", func() {
+				var (
+					siz = len(loremIpsum)
+					src = bytes.NewReader([]byte(loremIpsum)) // source
+					res = bytes.NewBuffer(make([]byte, 0))    // decompressed
+				)
+
+				// init new reader for the algo as compressor
+				c, e := archlp.New(algo, archlp.Compress, src)
+				Expect(e).NotTo(HaveOccurred())
+				Expect(c).NotTo(BeNil())
+
+				// init new reader for the algo as decompressor
+				d, e := archlp.New(algo, archlp.Decompress, c)
+				Expect(e).NotTo(HaveOccurred())
+				Expect(d).NotTo(BeNil())
+
+				// copy res data into buffer
+				n, e := io.Copy(res, d)
+				Expect(e).NotTo(HaveOccurred())
+				Expect(n).To(BeNumerically(">", 0))
+				Expect(n).To(BeNumerically("==", siz))
+
+				// closing reader
+				Expect(d.Close()).NotTo(HaveOccurred())
+				Expect(c.Close()).NotTo(HaveOccurred())
+
+				// check res must be same source
+				r := res.Bytes()
+				Expect(len(r)).To(BeNumerically("==", siz))
+				Expect(r).To(Equal([]byte(loremIpsum)))
+			})
+		})
+		Context("For the algo '"+algo.String()+"', in writer mode", func() {
+			It("should compress/decompress in embedded stream correctly", func() {
+				var (
+					siz = len(loremIpsum)
+					src = bytes.NewReader([]byte(loremIpsum)) // source
+					res = bytes.NewBuffer(make([]byte, 0))    // decompressed
+					wrt = bufio.NewWriter(res)
+					tmp *bufio.Writer
+				)
+
+				// init new reader for the algo as compressor
+				d, e := archlp.New(algo, archlp.Decompress, wrt)
+				Expect(e).NotTo(HaveOccurred())
+				Expect(d).NotTo(BeNil())
+				tmp = bufio.NewWriter(d)
+
+				// init new reader for the algo as compressor
+				c, e := archlp.New(algo, archlp.Compress, tmp)
+				Expect(e).NotTo(HaveOccurred())
+				Expect(c).NotTo(BeNil())
+
+				// copy res data into buffer
+				n, e := io.Copy(c, src)
+				Expect(e).NotTo(HaveOccurred())
+				Expect(n).To(BeNumerically(">", 0))
+				Expect(n).To(BeNumerically("==", siz))
+
+				// closing writer compressor and flush to decompressor
+				Expect(c.Close()).NotTo(HaveOccurred())
+				Expect(tmp.Flush()).To(Succeed())
+				Expect(d.Close()).NotTo(HaveOccurred())
+				Expect(wrt.Flush()).To(Succeed())
+
+				//need sleep to allow flush by applied on destination
+				time.Sleep(time.Second)
+
+				// check res must be same source
+				r := res.Bytes()
+				s := []byte(loremIpsum)
+				Expect(len(r)).To(BeNumerically("==", len(s)))
+				Expect(r).To(Equal(s))
+			})
+		})
+	}
+})

go.mod

@@ -2,25 +2,25 @@ module github.com/nabbar/golib
 
 go 1.23
 
-toolchain go1.23.2
+toolchain go1.23.3
 
 require (
 	github.com/aws/aws-sdk-go v1.55.5
-	github.com/aws/aws-sdk-go-v2 v1.32.0
-	github.com/aws/aws-sdk-go-v2/config v1.27.41
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.39
-	github.com/aws/aws-sdk-go-v2/service/iam v1.37.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0
-	github.com/aws/smithy-go v1.22.0
-	github.com/bits-and-blooms/bitset v1.14.3
+	github.com/aws/aws-sdk-go-v2 v1.32.5
+	github.com/aws/aws-sdk-go-v2/config v1.28.5
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.46
+	github.com/aws/aws-sdk-go-v2/service/iam v1.38.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.69.0
+	github.com/aws/smithy-go v1.22.1
+	github.com/bits-and-blooms/bitset v1.17.0
 	github.com/c-bata/go-prompt v0.2.6
 	github.com/dsnet/compress v0.0.1
-	github.com/fatih/color v1.17.0
-	github.com/fsnotify/fsnotify v1.7.0
+	github.com/fatih/color v1.18.0
+	github.com/fsnotify/fsnotify v1.8.0
 	github.com/fxamacker/cbor/v2 v2.7.0
 	github.com/gin-gonic/gin v1.10.0
 	github.com/go-ldap/ldap/v3 v3.4.8
-	github.com/go-playground/validator/v10 v10.22.1
+	github.com/go-playground/validator/v10 v10.23.0
 	github.com/google/go-github/v33 v33.0.0
 	github.com/hashicorp/go-hclog v1.6.3
 	github.com/hashicorp/go-retryablehttp v0.7.7
@@ -31,14 +31,14 @@ require (
 	github.com/mattn/go-colorable v0.1.13
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/nats-io/jwt/v2 v2.7.0
-	github.com/nats-io/nats-server/v2 v2.10.21
+	github.com/nats-io/jwt/v2 v2.7.2
+	github.com/nats-io/nats-server/v2 v2.10.22
 	github.com/nats-io/nats.go v1.37.0
-	github.com/onsi/ginkgo/v2 v2.20.2
-	github.com/onsi/gomega v1.34.2
+	github.com/onsi/ginkgo/v2 v2.22.0
+	github.com/onsi/gomega v1.36.0
 	github.com/pelletier/go-toml v1.9.5
 	github.com/pierrec/lz4/v4 v4.1.21
-	github.com/prometheus/client_golang v1.20.4
+	github.com/prometheus/client_golang v1.20.5
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.1
@@ -47,61 +47,61 @@ require (
 	github.com/ugorji/go/codec v1.2.12
 	github.com/ulikunitz/xz v0.5.12
 	github.com/vbauerster/mpb/v8 v8.8.3
-	github.com/xanzy/go-gitlab v0.110.0
+	github.com/xanzy/go-gitlab v0.114.0
 	github.com/xhit/go-simple-mail v2.2.2+incompatible
-	golang.org/x/net v0.30.0
-	golang.org/x/oauth2 v0.23.0
-	golang.org/x/sync v0.8.0
-	golang.org/x/sys v0.26.0
-	golang.org/x/term v0.25.0
+	golang.org/x/net v0.31.0
+	golang.org/x/oauth2 v0.24.0
+	golang.org/x/sync v0.9.0
+	golang.org/x/sys v0.27.0
+	golang.org/x/term v0.26.0
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/driver/clickhouse v0.6.1
 	gorm.io/driver/mysql v1.5.7
-	gorm.io/driver/postgres v1.5.9
+	gorm.io/driver/postgres v1.5.10
 	gorm.io/driver/sqlite v1.5.6
-	gorm.io/driver/sqlserver v1.5.3
+	gorm.io/driver/sqlserver v1.5.4
 	gorm.io/gorm v1.25.12
 )
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/ClickHouse/ch-go v0.62.0 // indirect
-	github.com/ClickHouse/clickhouse-go/v2 v2.29.0 // indirect
+	github.com/ClickHouse/ch-go v0.63.1 // indirect
+	github.com/ClickHouse/clickhouse-go/v2 v2.30.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/PuerkitoBio/goquery v1.10.0 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
-	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.19 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.19 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.32.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.24 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bytedance/sonic v1.12.3 // indirect
-	github.com/bytedance/sonic/loader v0.2.0 // indirect
+	github.com/bytedance/sonic v1.12.5 // indirect
+	github.com/bytedance/sonic/loader v0.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.5 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.7 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
 	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
-	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-sql-driver/mysql v1.8.1 // indirect
@@ -111,7 +111,7 @@ require (
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d // indirect
+	github.com/google/pprof v0.0.0-20241122213907-cbe949e5a41b // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/css v1.0.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -130,8 +130,8 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.10 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
@@ -145,7 +145,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nkeys v0.4.8 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/paulmach/orb v0.11.1 // indirect
@@ -153,7 +153,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pkg/term v1.2.0-beta.2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.60.0 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
@@ -169,18 +169,18 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/vanng822/css v1.0.1 // indirect
-	github.com/vanng822/go-premailer v1.21.0 // indirect
+	github.com/vanng822/go-premailer v1.22.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	go.opentelemetry.io/otel v1.30.0 // indirect
-	go.opentelemetry.io/otel/trace v1.30.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opentelemetry.io/otel v1.32.0 // indirect
+	go.opentelemetry.io/otel/trace v1.32.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/arch v0.11.0 // indirect
-	golang.org/x/crypto v0.28.0 // indirect
-	golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 // indirect
-	golang.org/x/text v0.19.0 // indirect
-	golang.org/x/time v0.7.0 // indirect
-	golang.org/x/tools v0.26.0 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	golang.org/x/arch v0.12.0 // indirect
+	golang.org/x/crypto v0.29.0 // indirect
+	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
+	golang.org/x/text v0.20.0 // indirect
+	golang.org/x/time v0.8.0 // indirect
+	golang.org/x/tools v0.27.0 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )