From 24e4191bc48a8ae2074ea11e1da7f45b337ff41c Mon Sep 17 00:00:00 2001 From: Nicolas JUHEL Date: Fri, 12 Apr 2024 16:32:03 +0200 Subject: [PATCH] Package artifact: - replace file progress to generic interface extended from io reader / writer with progress function - replace download function who's make the io copy, to a function that return the size + io.readCloser stream Package ioutils/ioprogress: - add a package to expose a generic interface based on io readcloser / writecloser - add a instance to create a io readcloser / writecloser with progress function - new instance based only on io readcloser/ writecloser and allow to register progress function or not if not wanted Other: - bump dependencies - adjust format licence for context/gin --- artifact/artifact.go | 4 +- artifact/github/model.go | 29 +++----- artifact/gitlab/model.go | 29 +++----- artifact/jfrog/model.go | 33 +++------ artifact/s3aws/model.go | 61 +++++------------ context/gin/interface.go | 39 ++++++----- context/gin/model.go | 46 +++++++------ go.mod | 99 ++++++++++++++------------- ioutils/ioprogress/interface.go | 71 +++++++++++++++++++ ioutils/ioprogress/reader.go | 116 ++++++++++++++++++++++++++++++++ ioutils/ioprogress/writer.go | 116 ++++++++++++++++++++++++++++++++ 11 files changed, 442 insertions(+), 201 deletions(-) create mode 100644 ioutils/ioprogress/interface.go create mode 100644 ioutils/ioprogress/reader.go create mode 100644 ioutils/ioprogress/writer.go diff --git a/artifact/artifact.go b/artifact/artifact.go index 0edf2c6..9983d18 100644 --- a/artifact/artifact.go +++ b/artifact/artifact.go @@ -26,6 +26,7 @@ package artifact import ( + "io" "os" "regexp" "strings" @@ -33,7 +34,6 @@ import ( hscvrs "github.com/hashicorp/go-version" artcli "github.com/nabbar/golib/artifact/client" liberr "github.com/nabbar/golib/errors" - libfpg "github.com/nabbar/golib/file/progress" ) const subUp = 20 @@ -50,7 +50,7 @@ type Client interface { ListReleases() (releases hscvrs.Collection, err error) GetArtifact(containName string, regexName string, release *hscvrs.Version) (link string, err error) - Download(dst libfpg.Progress, containName string, regexName string, release *hscvrs.Version) error + Download(containName string, regexName string, release *hscvrs.Version) (int64, io.ReadCloser, error) } func CheckRegex(name, regex string) bool { diff --git a/artifact/github/model.go b/artifact/github/model.go index 800531a..87535a0 100644 --- a/artifact/github/model.go +++ b/artifact/github/model.go @@ -36,7 +36,6 @@ import ( hscvrs "github.com/hashicorp/go-version" libart "github.com/nabbar/golib/artifact" artcli "github.com/nabbar/golib/artifact/client" - libfpg "github.com/nabbar/golib/file/progress" ) const ( @@ -112,46 +111,34 @@ func (g *githubModel) GetArtifact(containName string, regexName string, release return "", ErrorGithubNotFound.Error(nil) } -func (g *githubModel) Download(dst libfpg.Progress, containName string, regexName string, release *hscvrs.Version) error { +func (g *githubModel) Download(containName string, regexName string, release *hscvrs.Version) (int64, io.ReadCloser, error) { var ( uri string rsp *github.Response req *http.Request err error e error - n int64 ) defer func() { if req != nil && req.Body != nil { _ = req.Body.Close() } - if rsp != nil && rsp.Body != nil { - _ = rsp.Body.Close() - } }() if uri, e = g.GetArtifact(containName, regexName, release); e != nil { - return e + return 0, nil, e } else if req, err = g.c.NewRequest(http.MethodGet, uri, nil); err != nil { - return ErrorGithubRequestNew.Error(err) + return 0, nil, ErrorGithubRequestNew.Error(err) } else if rsp, err = g.c.Do(g.x, req, nil); err != nil { - return ErrorGithubRequestRun.Error(err) + return 0, nil, ErrorGithubRequestRun.Error(err) } else if rsp.StatusCode < 200 || rsp.StatusCode > 299 { - return ErrorGithubResponse.Error(errResponseCode) + return 0, nil, ErrorGithubResponse.Error(errResponseCode) } else if rsp.ContentLength < 1 { - return ErrorGithubResponse.Error(errResponseContents) + return 0, nil, ErrorGithubResponse.Error(errResponseContents) } else if rsp.Body == nil { - return ErrorGithubResponse.Error(errResponseBodyEmpty) + return 0, nil, ErrorGithubResponse.Error(errResponseBodyEmpty) } else { - dst.Reset(rsp.ContentLength) + return rsp.ContentLength, rsp.Body, nil } - - if n, err = io.Copy(dst, rsp.Body); err != nil { - return ErrorGithubIOCopy.Error(err) - } else if n != rsp.ContentLength { - return ErrorDestinationSize.Error(errMisMatchingSize) - } - - return nil } diff --git a/artifact/gitlab/model.go b/artifact/gitlab/model.go index 0d7aacc..7946a2f 100644 --- a/artifact/gitlab/model.go +++ b/artifact/gitlab/model.go @@ -36,7 +36,6 @@ import ( hscvrs "github.com/hashicorp/go-version" libart "github.com/nabbar/golib/artifact" artcli "github.com/nabbar/golib/artifact/client" - libfpg "github.com/nabbar/golib/file/progress" gitlab "github.com/xanzy/go-gitlab" ) @@ -111,46 +110,34 @@ func (g *gitlabModel) GetArtifact(containName string, regexName string, release return "", ErrorGitlabNotFound.Error(nil) } -func (g *gitlabModel) Download(dst libfpg.Progress, containName string, regexName string, release *hscvrs.Version) error { +func (g *gitlabModel) Download(containName string, regexName string, release *hscvrs.Version) (int64, io.ReadCloser, error) { var ( uri string rsp *gitlab.Response req *hschtc.Request err error e error - n int64 ) defer func() { if req != nil && req.Body != nil { _ = req.Body.Close() } - if rsp != nil && rsp.Body != nil { - _ = rsp.Body.Close() - } }() if uri, e = g.GetArtifact(containName, regexName, release); e != nil { - return e + return 0, nil, e } else if req, err = g.c.NewRequest(http.MethodGet, uri, nil, nil); err != nil { - return ErrorGitlabRequestNew.Error(err) + return 0, nil, ErrorGitlabRequestNew.Error(err) } else if rsp, err = g.c.Do(req, nil); err != nil { - return ErrorGitlabRequestRun.Error(err) + return 0, nil, ErrorGitlabRequestRun.Error(err) } else if rsp.StatusCode < 200 || rsp.StatusCode > 299 { - return ErrorGitlabResponse.Error(errResponseCode) + return 0, nil, ErrorGitlabResponse.Error(errResponseCode) } else if rsp.ContentLength < 1 { - return ErrorGitlabResponse.Error(errResponseContents) + return 0, nil, ErrorGitlabResponse.Error(errResponseContents) } else if rsp.Body == nil { - return ErrorGitlabResponse.Error(errResponseBodyEmpty) + return 0, nil, ErrorGitlabResponse.Error(errResponseBodyEmpty) } else { - dst.Reset(rsp.ContentLength) + return rsp.ContentLength, rsp.Body, nil } - - if n, err = io.Copy(dst, rsp.Body); err != nil { - return ErrorGitlabIOCopy.Error(err) - } else if n != rsp.ContentLength { - return ErrorDestinationSize.Error(errMisMatchingSize) - } - - return nil } diff --git a/artifact/jfrog/model.go b/artifact/jfrog/model.go index a7cb19a..b932567 100644 --- a/artifact/jfrog/model.go +++ b/artifact/jfrog/model.go @@ -42,7 +42,6 @@ import ( hscvrs "github.com/hashicorp/go-version" libart "github.com/nabbar/golib/artifact" artcli "github.com/nabbar/golib/artifact/client" - libfpg "github.com/nabbar/golib/file/progress" ) type artifactoryModel struct { @@ -319,10 +318,9 @@ func (a *artifactoryModel) GetArtifact(containName string, regexName string, rel } } -func (a *artifactoryModel) Download(dst libfpg.Progress, containName string, regexName string, release *hscvrs.Version) error { +func (a *artifactoryModel) Download(containName string, regexName string, release *hscvrs.Version) (int64, io.ReadCloser, error) { var ( e error - n int64 art *ResponseStorage err error @@ -331,38 +329,29 @@ func (a *artifactoryModel) Download(dst libfpg.Progress, containName string, reg ) defer func() { - if rsp != nil && rsp.Body != nil { - _ = rsp.Body.Close() - } - if req != nil && req.Body != nil { _ = req.Body.Close() } }() if art, err = a.getArtifact(containName, regexName, release); err != nil { - return err - } else { - dst.Reset(art.size) + return 0, nil, err } if req, e = http.NewRequestWithContext(a.ctx, http.MethodGet, art.DownloadUri, nil); e != nil { - return ErrorRequestInit.Error(e) + return 0, nil, ErrorRequestInit.Error(e) } else if rsp, e = a.Do(req); e != nil { - return ErrorRequestDo.Error(e) + return 0, nil, ErrorRequestDo.Error(e) } else if rsp.StatusCode >= http.StatusBadRequest { //nolint #goerr113 - return ErrorRequestResponse.Error(fmt.Errorf("status: %v", rsp.Status)) + return 0, nil, ErrorRequestResponse.Error(fmt.Errorf("status: %v", rsp.Status)) } else if rsp.Body == nil { //nolint #goerr113 - return ErrorRequestResponseBodyEmpty.Error(fmt.Errorf("status: %v", rsp.Status)) - } else if n, e = io.Copy(dst, rsp.Body); e != nil { - return ErrorArtifactoryDownload.Error(e) - } else if n != art.size { - return ErrorDestinationSize.Error(errMisMatchingSize) - } else if n != rsp.ContentLength { - return ErrorDestinationSize.Error(errMisMatchingSize) + return 0, nil, ErrorRequestResponseBodyEmpty.Error(fmt.Errorf("status: %v", rsp.Status)) + } else if art.size != rsp.ContentLength { + _ = rsp.Body.Close() + return 0, nil, ErrorDestinationSize.Error(errMisMatchingSize) + } else { + return rsp.ContentLength, rsp.Body, nil } - - return nil } diff --git a/artifact/s3aws/model.go b/artifact/s3aws/model.go index 055da9c..c1544ad 100644 --- a/artifact/s3aws/model.go +++ b/artifact/s3aws/model.go @@ -36,7 +36,6 @@ import ( libart "github.com/nabbar/golib/artifact" artcli "github.com/nabbar/golib/artifact/client" libaws "github.com/nabbar/golib/aws" - libfpg "github.com/nabbar/golib/file/progress" ) type s3awsModel struct { @@ -145,7 +144,7 @@ func (s *s3awsModel) GetArtifact(containName string, regexName string, release * return "", ErrorS3AWSNotFound.Error(getError(errVersRequest, release.String())) } -func (s *s3awsModel) Download(dst libfpg.Progress, containName string, regexName string, release *hscvrs.Version) error { +func (s *s3awsModel) Download(containName string, regexName string, release *hscvrs.Version) (int64, io.ReadCloser, error) { var ( e error r *regexp.Regexp @@ -157,11 +156,11 @@ func (s *s3awsModel) Download(dst libfpg.Progress, containName string, regexName ) if s.regex == "" { - return ErrorParamEmpty.Error(nil) + return 0, nil, ErrorParamEmpty.Error(nil) } - if l, err = s.c.Object().Find(s.regex); e != nil { - return ErrorS3AWSFind.Error(err) + if l, err = s.c.Object().Find(s.regex); err != nil { + return 0, nil, ErrorS3AWSFind.Error(err) } r = regexp.MustCompile(s.regex) @@ -170,68 +169,44 @@ func (s *s3awsModel) Download(dst libfpg.Progress, containName string, regexName grp := r.FindStringSubmatch(o) if len(grp) < s.group { - return ErrorS3AWSRegex.Error(getError(errRegexGroup, s.regex, len(grp), s.group)) + return 0, nil, ErrorS3AWSRegex.Error(getError(errRegexGroup, s.regex, len(grp), s.group)) } if v, e = hscvrs.NewVersion(grp[s.group]); e != nil { - return ErrorS3AWSNewVers.Error(getError(errVersion, grp[s.group]), e) + return 0, nil, ErrorS3AWSNewVers.Error(getError(errVersion, grp[s.group]), e) } else if v.Equal(release) { if containName != "" && strings.Contains(o, containName) { - return s.downloadObject(dst, o) + return s.downloadObject(o) } if regexName != "" { if k, e = regexp.MatchString(regexName, o); e == nil && k { - return s.downloadObject(dst, o) + return s.downloadObject(o) } } if containName == "" && regexName == "" { - return s.downloadObject(dst, o) + return s.downloadObject(o) } } } - return ErrorS3AWSNotFound.Error(getError(errVersRequest, release.String())) + return 0, nil, ErrorS3AWSNotFound.Error(getError(errVersRequest, release.String())) } -func (s *s3awsModel) downloadObject(dst libfpg.Progress, object string) error { +func (s *s3awsModel) downloadObject(object string) (int64, io.ReadCloser, error) { var ( - r *sdksss.GetObjectOutput - e error - j int64 - n int64 - + r *sdksss.GetObjectOutput err error ) - defer func() { - if r != nil && r.Body != nil { - _ = r.Body.Close() - } - }() - - if j, err = s.c.Object().Size(object); err != nil { - er := ErrorS3AWSDownloadError.Error(getError(errObject, object)) - er.Add(err) - return er - } else if j < 1 { - return ErrorS3AWSDownloadError.Error(getError(errObjectEmpty, object)) - } else { - dst.Reset(j) - } - if r, err = s.c.Object().Get(object); err != nil { - er := ErrorS3AWSDownloadError.Error(getError(errObject, object)) - er.Add(err) - return er + return 0, nil, ErrorS3AWSDownloadError.Error(getError(errObject, object), err) + } else if r.ContentLength == nil || *r.ContentLength < 1 { + return 0, nil, ErrorS3AWSDownloadError.Error(getError(errObjectEmpty, object)) } else if r.Body == nil { - return ErrorS3AWSIOReaderError.Error(getError(errObject, object)) - } else if n, e = io.Copy(dst, r.Body); e != nil { - return ErrorS3AWSDownloadError.Error(getError(errObject, object), e) - } else if n != j { - return ErrorS3AWSDownloadError.Error(getError(errObjectSize, object)) + return 0, nil, ErrorS3AWSIOReaderError.Error(getError(errObject, object)) + } else { + return *r.ContentLength, r.Body, nil } - - return nil } diff --git a/context/gin/interface.go b/context/gin/interface.go index c6241b6..68e40bb 100644 --- a/context/gin/interface.go +++ b/context/gin/interface.go @@ -1,29 +1,28 @@ -/*********************************************************************************************************************** +/* + * MIT License * - * MIT License + * Copyright (c) 2019 Nicolas JUHEL * - * Copyright (c) 2022 Nicolas JUHEL + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. * * - **********************************************************************************************************************/ + */ package gin diff --git a/context/gin/model.go b/context/gin/model.go index ee2b52e..0009b1c 100644 --- a/context/gin/model.go +++ b/context/gin/model.go @@ -1,26 +1,28 @@ /* -MIT License - -Copyright (c) 2019 Nicolas JUHEL - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ + * MIT License + * + * Copyright (c) 2019 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ package gin diff --git a/go.mod b/go.mod index 95031be..ade5260 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,17 @@ module github.com/nabbar/golib -go 1.21 +go 1.22 -toolchain go1.22.1 +toolchain go1.22.2 require ( - github.com/aws/aws-sdk-go v1.51.6 - github.com/aws/aws-sdk-go-v2 v1.26.0 - github.com/aws/aws-sdk-go-v2/config v1.27.9 - github.com/aws/aws-sdk-go-v2/credentials v1.17.9 - github.com/aws/aws-sdk-go-v2/service/iam v1.31.3 - github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 - github.com/aws/smithy-go v1.20.1 + github.com/aws/aws-sdk-go v1.51.21 + github.com/aws/aws-sdk-go-v2 v1.26.1 + github.com/aws/aws-sdk-go-v2/config v1.27.11 + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 + github.com/aws/aws-sdk-go-v2/service/iam v1.32.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 + github.com/aws/smithy-go v1.20.2 github.com/bits-and-blooms/bitset v1.13.0 github.com/c-bata/go-prompt v0.2.6 github.com/dsnet/compress v0.0.1 @@ -19,10 +19,10 @@ require ( github.com/fsnotify/fsnotify v1.7.0 github.com/fxamacker/cbor/v2 v2.6.0 github.com/gin-gonic/gin v1.9.1 - github.com/go-ldap/ldap/v3 v3.4.6 + github.com/go-ldap/ldap/v3 v3.4.7 github.com/go-playground/validator/v10 v10.19.0 github.com/google/go-github/v33 v33.0.0 - github.com/hashicorp/go-hclog v1.6.2 + github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-retryablehttp v0.7.5 github.com/hashicorp/go-uuid v1.0.3 github.com/hashicorp/go-version v1.6.0 @@ -33,8 +33,8 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.5.0 github.com/nats-io/jwt/v2 v2.5.5 - github.com/nats-io/nats-server/v2 v2.10.12 - github.com/nats-io/nats.go v1.34.0 + github.com/nats-io/nats-server/v2 v2.10.14 + github.com/nats-io/nats.go v1.34.1 github.com/nutsdb/nutsdb v0.14.3 github.com/onsi/ginkgo/v2 v2.17.1 github.com/onsi/gomega v1.32.0 @@ -47,31 +47,31 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 github.com/spf13/viper v1.18.2 github.com/ugorji/go/codec v1.2.12 - github.com/ulikunitz/xz v0.5.11 - github.com/vbauerster/mpb/v8 v8.7.2 - github.com/xanzy/go-gitlab v0.101.0 + github.com/ulikunitz/xz v0.5.12 + github.com/vbauerster/mpb/v8 v8.7.3 + github.com/xanzy/go-gitlab v0.102.0 github.com/xhit/go-simple-mail v2.2.2+incompatible github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 - golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 - golang.org/x/net v0.22.0 - golang.org/x/oauth2 v0.18.0 - golang.org/x/sync v0.6.0 - golang.org/x/sys v0.18.0 - golang.org/x/term v0.18.0 + golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 + golang.org/x/net v0.24.0 + golang.org/x/oauth2 v0.19.0 + golang.org/x/sync v0.7.0 + golang.org/x/sys v0.19.0 + golang.org/x/term v0.19.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/clickhouse v0.6.0 gorm.io/driver/mysql v1.5.6 gorm.io/driver/postgres v1.5.7 gorm.io/driver/sqlite v1.5.5 gorm.io/driver/sqlserver v1.5.3 - gorm.io/gorm v1.25.8 + gorm.io/gorm v1.25.9 ) require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/ClickHouse/ch-go v0.61.5 // indirect - github.com/ClickHouse/clickhouse-go/v2 v2.22.2 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.23.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible // indirect @@ -84,25 +84,25 @@ require ( github.com/antlabs/stl v0.0.2 // indirect github.com/antlabs/timer v0.1.3 // indirect github.com/armon/go-metrics v0.4.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 // indirect - github.com/bytedance/sonic v1.11.3 // indirect + github.com/bytedance/sonic v1.11.4 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect - github.com/chenzhuoyu/iasm v0.9.1 // indirect + github.com/cloudwego/base64x v0.1.0 // indirect + github.com/cloudwego/iasm v0.0.9 // indirect github.com/cockroachdb/errors v1.7.5 // indirect github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb // indirect @@ -117,19 +117,19 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-sql-driver/mysql v1.8.0 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/pprof v0.0.0-20240320155624-b11c3daa6f07 // indirect + github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -154,7 +154,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/ratelimit v1.0.2-0.20191002062651-f60b32039441 // indirect - github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -176,7 +176,7 @@ require ( github.com/nats-io/nuid v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/paulmach/orb v0.11.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/term v1.2.0-beta.2 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -189,7 +189,7 @@ require ( github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/segmentio/asm v1.2.0 // indirect - github.com/shopspring/decimal v1.3.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect @@ -205,15 +205,14 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/xujiajun/mmap-go v1.0.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opentelemetry.io/otel v1.25.0 // indirect + go.opentelemetry.io/otel/trace v1.25.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.7.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.19.0 // indirect - google.golang.org/appengine v1.6.8 // indirect + golang.org/x/tools v0.20.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/ioutils/ioprogress/interface.go b/ioutils/ioprogress/interface.go new file mode 100644 index 0000000..fc4929a --- /dev/null +++ b/ioutils/ioprogress/interface.go @@ -0,0 +1,71 @@ +/* + * MIT License + * + * Copyright (c) 2024 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package ioprogress + +import ( + "io" + "sync/atomic" + + libfpg "github.com/nabbar/golib/file/progress" +) + +type Progress interface { + RegisterFctIncrement(fct libfpg.FctIncrement) + RegisterFctReset(fct libfpg.FctReset) + RegisterFctEOF(fct libfpg.FctEOF) + Reset(max int64) +} + +type Reader interface { + io.ReadCloser + Progress +} + +type Writer interface { + io.WriteCloser + Progress +} + +func NewReadCloser(r io.ReadCloser) Reader { + return &rdr{ + r: r, + cr: new(atomic.Int64), + fi: new(atomic.Value), + fe: new(atomic.Value), + fr: new(atomic.Value), + } +} + +func NewWriteCloser(w io.WriteCloser) Writer { + return &wrt{ + w: w, + cr: new(atomic.Int64), + fi: new(atomic.Value), + fe: new(atomic.Value), + fr: new(atomic.Value), + } +} diff --git a/ioutils/ioprogress/reader.go b/ioutils/ioprogress/reader.go new file mode 100644 index 0000000..24ec16d --- /dev/null +++ b/ioutils/ioprogress/reader.go @@ -0,0 +1,116 @@ +/* + * MIT License + * + * Copyright (c) 2024 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package ioprogress + +import ( + "errors" + "io" + "sync/atomic" + + libfpg "github.com/nabbar/golib/file/progress" +) + +type rdr struct { + r io.ReadCloser + + cr *atomic.Int64 + fi *atomic.Value + fe *atomic.Value + fr *atomic.Value +} + +func (r *rdr) Read(p []byte) (n int, err error) { + n, err = r.r.Read(p) + r.inc(n) + + if errors.Is(err, io.EOF) { + r.finish() + } + + return n, err +} + +func (r *rdr) Close() error { + return r.r.Close() +} + +func (r *rdr) RegisterFctIncrement(fct libfpg.FctIncrement) { + if fct == nil { + fct = func(size int64) {} + } + + r.fi.Store(fct) +} + +func (r *rdr) RegisterFctReset(fct libfpg.FctReset) { + if fct == nil { + fct = func(size, current int64) {} + } + + r.fr.Store(fct) +} + +func (r *rdr) RegisterFctEOF(fct libfpg.FctEOF) { + if fct == nil { + fct = func() {} + } + + r.fe.Store(fct) +} + +func (r *rdr) inc(n int) { + if r == nil { + return + } + + r.cr.Add(int64(n)) + + f := r.fi.Load() + if f != nil { + f.(libfpg.FctIncrement)(int64(n)) + } +} + +func (r *rdr) finish() { + if r == nil { + return + } + + f := r.fe.Load() + if f != nil { + f.(libfpg.FctEOF)() + } +} + +func (r *rdr) Reset(max int64) { + if r == nil { + return + } + + f := r.fr.Load() + f.(libfpg.FctReset)(max, r.cr.Load()) +} diff --git a/ioutils/ioprogress/writer.go b/ioutils/ioprogress/writer.go new file mode 100644 index 0000000..9d4ade6 --- /dev/null +++ b/ioutils/ioprogress/writer.go @@ -0,0 +1,116 @@ +/* + * MIT License + * + * Copyright (c) 2024 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package ioprogress + +import ( + "errors" + "io" + "sync/atomic" + + libfpg "github.com/nabbar/golib/file/progress" +) + +type wrt struct { + w io.WriteCloser + + cr *atomic.Int64 + fi *atomic.Value + fe *atomic.Value + fr *atomic.Value +} + +func (w *wrt) Write(p []byte) (n int, err error) { + n, err = w.w.Write(p) + w.inc(n) + + if errors.Is(err, io.EOF) { + w.finish() + } + + return n, err +} + +func (w *wrt) Close() error { + return w.w.Close() +} + +func (w *wrt) RegisterFctIncrement(fct libfpg.FctIncrement) { + if fct == nil { + fct = func(size int64) {} + } + + w.fi.Store(fct) +} + +func (w *wrt) RegisterFctReset(fct libfpg.FctReset) { + if fct == nil { + fct = func(size, current int64) {} + } + + w.fr.Store(fct) +} + +func (w *wrt) RegisterFctEOF(fct libfpg.FctEOF) { + if fct == nil { + fct = func() {} + } + + w.fe.Store(fct) +} + +func (w *wrt) inc(n int) { + if w == nil { + return + } + + w.cr.Add(int64(n)) + + f := w.fi.Load() + if f != nil { + f.(libfpg.FctIncrement)(int64(n)) + } +} + +func (w *wrt) finish() { + if w == nil { + return + } + + f := w.fe.Load() + if f != nil { + f.(libfpg.FctEOF)() + } +} + +func (w *wrt) Reset(max int64) { + if w == nil { + return + } + + f := w.fr.Load() + f.(libfpg.FctReset)(max, w.cr.Load()) +}