diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 31ea971..66474e6 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -73,6 +73,8 @@ jobs: max_attempts: 3 timeout_minutes: 15 command: | + wget --quiet --output-document="aws/minio" "https://dl.min.io/server/minio/release/linux-amd64/minio" + chmod -v +x "aws/minio" go version ginkgo version for PKG in $(find $(pwd) -type f -name "*_suite_test.go" | sort -u ); diff --git a/.gitignore b/.gitignore index 6992440..c72f392 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,10 @@ go.sum .idea/* .idea/**/* +# remove minio from aws package +aws/minio + +# other /scripts /test-* diff --git a/archive/archive.go b/archive/archive.go index 4dd5948..e5dfd11 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -36,11 +36,16 @@ import ( libtar "github.com/nabbar/golib/archive/tar" libzip "github.com/nabbar/golib/archive/zip" liberr "github.com/nabbar/golib/errors" - libiot "github.com/nabbar/golib/ioutils" + libfpg "github.com/nabbar/golib/file/progress" ) type ArchiveType uint8 +const ( + permDir = os.FileMode(0775) + permFile = os.FileMode(0664) +) + const ( TypeTar = iota + 1 TypeTarGzip @@ -48,17 +53,27 @@ const ( TypeZip ) -func ExtractFile(src, dst libiot.FileProgress, fileNameContain, fileNameRegex string) liberr.Error { +func ExtractFile(src, dst libfpg.Progress, fileNameContain, fileNameRegex string) liberr.Error { var ( - tmp libiot.FileProgress + e error + + tmp libfpg.Progress err liberr.Error ) - if tmp, err = dst.NewFileTemp(); err != nil { - return err + defer func() { + if tmp != nil { + _ = tmp.CloseDelete() + } + }() + + if tmp, e = libfpg.Temp(""); e != nil { + return ErrorFileOpen.ErrorParent(e) + } else { + dst.SetRegisterProgress(tmp) } - if _, e := src.Seek(0, io.SeekStart); e != nil { + if _, e = src.Seek(0, io.SeekStart); e != nil { return ErrorFileSeek.ErrorParent(e) // #nosec } @@ -91,9 +106,7 @@ func ExtractFile(src, dst libiot.FileProgress, fileNameContain, fileNameRegex st return err } - _ = tmp.Close() - - if _, e := dst.ReadFrom(src); e != nil { + if _, e = dst.ReadFrom(src); e != nil { //logger.ErrorLevel.LogErrorCtx(logger.DebugLevel, "reopening file", err) return ErrorIOCopy.ErrorParent(e) } @@ -101,10 +114,13 @@ func ExtractFile(src, dst libiot.FileProgress, fileNameContain, fileNameRegex st return nil } -func ExtractAll(src libiot.FileProgress, originalName, outputPath string, defaultDirPerm os.FileMode) liberr.Error { +func ExtractAll(src libfpg.Progress, originalName, outputPath string, defaultDirPerm os.FileMode) liberr.Error { var ( - tmp libiot.FileProgress - dst libiot.FileProgress + e error + i os.FileInfo + + tmp libfpg.Progress + dst libfpg.Progress err liberr.Error ) @@ -116,12 +132,14 @@ func ExtractAll(src libiot.FileProgress, originalName, outputPath string, defaul _ = dst.Close() } if tmp != nil { - _ = tmp.Close() + _ = tmp.CloseDelete() } }() - if tmp, err = src.NewFileTemp(); err != nil { - return ErrorFileOpen.Error(err) + if tmp, e = libfpg.Temp(""); e != nil { + return ErrorFileOpen.ErrorParent(e) + } else { + src.SetRegisterProgress(tmp) } if err = libbz2.GetFile(src, tmp); err == nil { @@ -140,10 +158,10 @@ func ExtractAll(src libiot.FileProgress, originalName, outputPath string, defaul _ = tmp.Close() } - if i, e := os.Stat(outputPath); e != nil && os.IsNotExist(e) { + if i, e = os.Stat(outputPath); e != nil && os.IsNotExist(e) { //nolint #nosec /* #nosec */ - if e := os.MkdirAll(outputPath, 0775); e != nil { + if e = os.MkdirAll(outputPath, permDir); e != nil { return ErrorDirCreate.ErrorParent(e) } } else if e != nil { @@ -164,20 +182,22 @@ func ExtractAll(src libiot.FileProgress, originalName, outputPath string, defaul return err } - if dst, err = src.NewFilePathWrite(filepath.Join(outputPath, originalName), true, true, 0664); err != nil { - return ErrorFileOpen.Error(err) + if dst, e = libfpg.New(filepath.Join(outputPath, originalName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, permFile); e != nil { + return ErrorFileOpen.ErrorParent(e) + } else { + src.SetRegisterProgress(dst) } - if _, e := src.Seek(0, io.SeekStart); e != nil { + if _, e = src.Seek(0, io.SeekStart); e != nil { return ErrorFileSeek.ErrorParent(e) - } else if _, e := dst.ReadFrom(src); e != nil { + } else if _, e = dst.ReadFrom(src); e != nil { return ErrorIOCopy.ErrorParent(e) } return nil } -func CreateArchive(archiveType ArchiveType, archive libiot.FileProgress, stripPath string, comment string, pathContent ...string) (created bool, err liberr.Error) { +func CreateArchive(archiveType ArchiveType, archive libfpg.Progress, stripPath string, comment string, pathContent ...string) (created bool, err liberr.Error) { if len(pathContent) < 1 { //nolint #goerr113 return false, ErrorParamEmpty.ErrorParent(fmt.Errorf("pathContent is empty")) diff --git a/archive/bz2/error.go b/archive/bz2/error.go index 7ead9f9..b3b0f67 100644 --- a/archive/bz2/error.go +++ b/archive/bz2/error.go @@ -33,6 +33,8 @@ import ( liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/archive/bz2" + const ( ErrorParamEmpty liberr.CodeError = iota + arcmod.MinPkgArchiveBZ2 ErrorFileSeek @@ -41,13 +43,15 @@ const ( func init() { if liberr.ExistInMapMessage(ErrorParamEmpty) { - panic(fmt.Errorf("error code collision golib/archive/bz2")) + panic(fmt.Errorf("error code collision %s", pkgName)) } liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } func getMessage(code liberr.CodeError) (message string) { switch code { + case liberr.UnknownError: + return liberr.NullMessage case ErrorParamEmpty: return "given parameters is empty" case ErrorFileSeek: diff --git a/archive/error.go b/archive/error.go index fe8c021..2204279 100644 --- a/archive/error.go +++ b/archive/error.go @@ -33,6 +33,8 @@ import ( liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/archive" + const ( ErrorParamEmpty liberr.CodeError = iota + arcmod.MinPkgArchive ErrorFileSeek @@ -46,13 +48,15 @@ const ( func init() { if liberr.ExistInMapMessage(ErrorParamEmpty) { - panic(fmt.Errorf("error code collision golib/archive")) + panic(fmt.Errorf("error code collision %s", pkgName)) } liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } func getMessage(code liberr.CodeError) (message string) { switch code { + case liberr.UnknownError: + return liberr.NullMessage case ErrorParamEmpty: return "given parameters is empty" case ErrorFileSeek: diff --git a/archive/gzip/error.go b/archive/gzip/error.go index 279caba..7b22f09 100644 --- a/archive/gzip/error.go +++ b/archive/gzip/error.go @@ -33,6 +33,8 @@ import ( liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/archive/gzip" + const ( ErrorParamEmpty liberr.CodeError = iota + arcmod.MinPkgArchiveGZip ErrorParamMismatching @@ -45,13 +47,15 @@ const ( func init() { if liberr.ExistInMapMessage(ErrorParamEmpty) { - panic(fmt.Errorf("error code collision golib/archive/gzip")) + panic(fmt.Errorf("error code collision %s", pkgName)) } liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } func getMessage(code liberr.CodeError) (message string) { switch code { + case liberr.UnknownError: + return liberr.NullMessage case ErrorParamEmpty: return "given parameters is empty" case ErrorParamMismatching: diff --git a/archive/gzipreader/interface.go b/archive/gzipreader/interface.go new file mode 100644 index 0000000..c06bce6 --- /dev/null +++ b/archive/gzipreader/interface.go @@ -0,0 +1,54 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package gzipreader + +import ( + "bytes" + "compress/gzip" + "io" +) + +type GZipReader interface { + io.Reader + + LenCompressed() int64 + LenUnCompressed() int64 + + Rate() float64 +} + +// GzipReader is used to GZIP a io.reader on fly. +// The given io.reader is not a gzip reader but the result is a GZipped reader +func GzipReader(r io.Reader) GZipReader { + b := bytes.NewBuffer(make([]byte, 0, 32*1024)) + + return &gzr{ + r: r, + b: b, + z: gzip.NewWriter(b), + } +} diff --git a/archive/gzipreader/reader.go b/archive/gzipreader/reader.go new file mode 100644 index 0000000..236d0c3 --- /dev/null +++ b/archive/gzipreader/reader.go @@ -0,0 +1,97 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package gzipreader + +import ( + "bytes" + "compress/gzip" + "errors" + "io" +) + +type gzr struct { + r io.Reader + b *bytes.Buffer + z *gzip.Writer + nc int64 + nu int64 +} + +func (o *gzr) Read(p []byte) (n int, err error) { + var ( + s []byte + + er error + nr int + + ew error + nw int + ) + + if i := cap(p); i > o.b.Cap() || i < 1 { + s = make([]byte, 0, o.b.Cap()) + } else { + s = make([]byte, 0, i) + } + + nr, er = o.r.Read(s) + o.nu += int64(nr) + + if er != nil && !errors.Is(er, io.EOF) { + return 0, err + } else if nr > 0 { + nw, ew = o.z.Write(s) + } + + if ew != nil { + return 0, ew + } else if nw != nr { + return 0, errors.New("invalid write buffer") + } else if er != nil && errors.Is(er, io.EOF) { + if ew = o.z.Close(); ew != nil { + return 0, ew + } + } + + copy(p, o.b.Bytes()) + o.b.Reset() + o.nc += int64(len(p)) + + return len(p), er +} + +func (o *gzr) LenCompressed() int64 { + return o.nc +} + +func (o *gzr) LenUnCompressed() int64 { + return o.nu +} + +func (o *gzr) Rate() float64 { + return 1 - float64(o.nc/o.nu) +} diff --git a/archive/tar/error.go b/archive/tar/error.go index 8c6a1d8..96c17b0 100644 --- a/archive/tar/error.go +++ b/archive/tar/error.go @@ -33,6 +33,8 @@ import ( liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/archive/tar" + const ( ErrorParamEmpty liberr.CodeError = iota + arcmod.MinPkgArchiveTar ErrorTarNext @@ -54,13 +56,15 @@ const ( func init() { if liberr.ExistInMapMessage(ErrorParamEmpty) { - panic(fmt.Errorf("error code collision golib/archive/tar")) + panic(fmt.Errorf("error code collision %s", pkgName)) } liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } func getMessage(code liberr.CodeError) (message string) { switch code { + case liberr.UnknownError: + return liberr.NullMessage case ErrorParamEmpty: return "given parameters is empty" case ErrorTarNext: diff --git a/archive/tar/reader.go b/archive/tar/reader.go index c6ccf08..6c5c8bc 100644 --- a/archive/tar/reader.go +++ b/archive/tar/reader.go @@ -33,12 +33,13 @@ import ( "runtime" "strings" + libfpg "github.com/nabbar/golib/file/progress" + libarc "github.com/nabbar/golib/archive/archive" liberr "github.com/nabbar/golib/errors" - libiot "github.com/nabbar/golib/ioutils" ) -func GetFile(src, dst libiot.FileProgress, filenameContain, filenameRegex string) liberr.Error { +func GetFile(src, dst libfpg.Progress, filenameContain, filenameRegex string) liberr.Error { if _, e := src.Seek(0, io.SeekStart); e != nil { return ErrorFileSeek.ErrorParent(e) @@ -102,8 +103,10 @@ func GetAll(src io.ReadSeeker, outputFolder string, defaultDirPerm os.FileMode) func writeContent(r io.Reader, h *tar.Header, out string, defaultDirPerm os.FileMode) (err liberr.Error) { var ( + e error + inf = h.FileInfo() - dst libiot.FileProgress + dst libfpg.Progress ) if e := dirIsExistOrCreate(filepath.Dir(out), defaultDirPerm); e != nil { @@ -130,9 +133,9 @@ func writeContent(r io.Reader, h *tar.Header, out string, defaultDirPerm os.File return createLink(out, libarc.CleanPath(h.Linkname), true) } - if dst, err = libiot.NewFileProgressPathWrite(out, true, true, inf.Mode()); err != nil { - return ErrorFileOpen.Error(err) - } else if _, e := io.Copy(dst, r); e != nil { + if dst, e = libfpg.New(out, os.O_RDWR|os.O_CREATE|os.O_TRUNC, inf.Mode()); e != nil { + return ErrorFileOpen.ErrorParent(e) + } else if _, e = io.Copy(dst, r); e != nil { return ErrorIOCopy.ErrorParent(e) } else if e = dst.Close(); e != nil { return ErrorFileClose.ErrorParent(e) diff --git a/archive/zip/error.go b/archive/zip/error.go index c2af2b4..901f1df 100644 --- a/archive/zip/error.go +++ b/archive/zip/error.go @@ -33,6 +33,8 @@ import ( liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/archive/zip" + const ( ErrorParamEmpty liberr.CodeError = iota + arcmod.MinPkgArchiveZip ErrorFileOpen @@ -55,13 +57,15 @@ const ( func init() { if liberr.ExistInMapMessage(ErrorParamEmpty) { - panic(fmt.Errorf("error code collision golib/archive/zip")) + panic(fmt.Errorf("error code collision %s", pkgName)) } liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } func getMessage(code liberr.CodeError) (message string) { switch code { + case liberr.UnknownError: + return liberr.NullMessage case ErrorParamEmpty: return "given parameters is empty" case ErrorFileOpen: diff --git a/archive/zip/reader.go b/archive/zip/reader.go index 9cc61d9..70de2d2 100644 --- a/archive/zip/reader.go +++ b/archive/zip/reader.go @@ -31,12 +31,13 @@ import ( "os" "path/filepath" + libfpg "github.com/nabbar/golib/file/progress" + arcmod "github.com/nabbar/golib/archive/archive" liberr "github.com/nabbar/golib/errors" - libiot "github.com/nabbar/golib/ioutils" ) -func GetFile(src, dst libiot.FileProgress, filenameContain, filenameRegex string) liberr.Error { +func GetFile(src, dst libfpg.Progress, filenameContain, filenameRegex string) liberr.Error { var ( arc *zip.Reader inf os.FileInfo @@ -47,7 +48,7 @@ func GetFile(src, dst libiot.FileProgress, filenameContain, filenameRegex string return ErrorFileSeek.ErrorParent(err) } else if _, err = dst.Seek(0, io.SeekStart); err != nil { return ErrorFileSeek.ErrorParent(err) - } else if inf, err = src.FileStat(); err != nil { + } else if inf, err = src.Stat(); err != nil { return ErrorFileStat.ErrorParent(err) } else if arc, err = zip.NewReader(src, inf.Size()); err != nil { return ErrorZipOpen.ErrorParent(err) @@ -98,7 +99,7 @@ func GetFile(src, dst libiot.FileProgress, filenameContain, filenameRegex string return nil } -func GetAll(src libiot.FileProgress, outputFolder string, defaultDirPerm os.FileMode) liberr.Error { +func GetAll(src libfpg.Progress, outputFolder string, defaultDirPerm os.FileMode) liberr.Error { var ( r *zip.Reader i os.FileInfo @@ -107,7 +108,7 @@ func GetAll(src libiot.FileProgress, outputFolder string, defaultDirPerm os.File if _, e = src.Seek(0, io.SeekStart); e != nil { return ErrorFileSeek.ErrorParent(e) - } else if i, e = src.FileStat(); e != nil { + } else if i, e = src.Stat(); e != nil { return ErrorFileStat.ErrorParent(e) } else if r, e = zip.NewReader(src, i.Size()); e != nil { return ErrorZipOpen.ErrorParent(e) @@ -130,7 +131,7 @@ func GetAll(src libiot.FileProgress, outputFolder string, defaultDirPerm os.File func writeContent(f *zip.File, out string, defaultDirPerm os.FileMode) (err liberr.Error) { var ( - dst libiot.FileProgress + dst libfpg.Progress inf = f.FileInfo() r io.ReadCloser @@ -164,9 +165,13 @@ func writeContent(f *zip.File, out string, defaultDirPerm os.FileMode) (err libe return } - if dst, err = libiot.NewFileProgressPathWrite(out, true, true, inf.Mode()); err != nil { - return ErrorFileOpen.Error(err) - } else if r, e = f.Open(); e != nil { + if dst, e = libfpg.New(out, os.O_RDWR|os.O_CREATE|os.O_TRUNC, inf.Mode()); e != nil { + return ErrorFileOpen.ErrorParent(e) + } else { + + } + + if r, e = f.Open(); e != nil { return ErrorZipFileOpen.ErrorParent(e) } diff --git a/artifact/artifact.go b/artifact/artifact.go index 94ebbfa..ed6a2d5 100644 --- a/artifact/artifact.go +++ b/artifact/artifact.go @@ -30,26 +30,27 @@ import ( "regexp" "strings" - "github.com/hashicorp/go-version" - "github.com/nabbar/golib/artifact/client" - "github.com/nabbar/golib/errors" - "github.com/nabbar/golib/ioutils" + hscvrs "github.com/hashicorp/go-version" + artcli "github.com/nabbar/golib/artifact/client" + liberr "github.com/nabbar/golib/errors" + libfpg "github.com/nabbar/golib/file/progress" ) +const subUp = 20 + const ( - MIN_ARTIFACT_ARTIFAC = errors.MinPkgArtifact + 10 - MIN_ARTIFACT_GITLAB = errors.MinPkgArtifact + 20 - MIN_ARTIFACT_GITHUB = errors.MinPkgArtifact + 40 - MIN_ARTIFACT_JFORG = errors.MinPkgArtifact + 60 - MIN_ARTIFACT_S3AWS = errors.MinPkgArtifact + 80 + MinArtifactGitlab = subUp + liberr.MinPkgArtifact + MinArtifactGithub = subUp + MinArtifactGitlab + MinArtifactJfrog = subUp + MinArtifactGithub + MinArtifactS3AWS = subUp + MinArtifactJfrog ) type Client interface { - client.ArtifactManagement + artcli.ArtifactManagement - ListReleases() (releases version.Collection, err errors.Error) - GetArtifact(containName string, regexName string, release *version.Version) (link string, err errors.Error) - Download(dst ioutils.FileProgress, containName string, regexName string, release *version.Version) errors.Error + ListReleases() (releases hscvrs.Collection, err liberr.Error) + GetArtifact(containName string, regexName string, release *hscvrs.Version) (link string, err liberr.Error) + Download(dst libfpg.Progress, containName string, regexName string, release *hscvrs.Version) liberr.Error } func CheckRegex(name, regex string) bool { @@ -60,25 +61,29 @@ func CheckRegex(name, regex string) bool { return false } -func DownloadRelease(link string) (file os.File, err errors.Error) { +func DownloadRelease(link string) (file os.File, err liberr.Error) { panic("not implemented") } -func ValidatePreRelease(version *version.Version) bool { - p := strings.ToLower(version.Prerelease()) +func ValidatePreRelease(version *hscvrs.Version) bool { + var ( + p = strings.ToLower(version.Prerelease()) + s = []string{ + "a", "alpha", + "b", "beta", + "rc", + "dev", + "test", + "draft", + "master", + "main", + } + ) - if strings.Contains(p, "alpha") { - return false - } else if strings.Contains(p, "beta") { - return false - } else if strings.Contains(p, "rc") { - return false - } else if strings.Contains(p, "dev") { - return false - } else if strings.Contains(p, "test") { - return false - } else if strings.Contains(p, "master") { - return false + for _, i := range s { + if strings.Contains(p, i) { + return false + } } return true diff --git a/artifact/client/interface.go b/artifact/client/interface.go index c3fa844..9081552 100644 --- a/artifact/client/interface.go +++ b/artifact/client/interface.go @@ -26,16 +26,16 @@ package client import ( - "github.com/hashicorp/go-version" - "github.com/nabbar/golib/errors" + hscvrs "github.com/hashicorp/go-version" + liberr "github.com/nabbar/golib/errors" ) type ArtifactManagement interface { - ListReleasesOrder() (releases map[int]map[int]version.Collection, err errors.Error) - ListReleasesMajor(major int) (releases version.Collection, err errors.Error) - ListReleasesMinor(major, minor int) (releases version.Collection, err errors.Error) + ListReleasesOrder() (releases map[int]map[int]hscvrs.Collection, err liberr.Error) + ListReleasesMajor(major int) (releases hscvrs.Collection, err liberr.Error) + ListReleasesMinor(major, minor int) (releases hscvrs.Collection, err liberr.Error) - GetLatest() (release *version.Version, err errors.Error) - GetLatestMajor(major int) (release *version.Version, err errors.Error) - GetLatestMinor(major, minor int) (release *version.Version, err errors.Error) + GetLatest() (release *hscvrs.Version, err liberr.Error) + GetLatestMajor(major int) (release *hscvrs.Version, err liberr.Error) + GetLatestMinor(major, minor int) (release *hscvrs.Version, err liberr.Error) } diff --git a/artifact/client/model.go b/artifact/client/model.go index 5920592..a3cb549 100644 --- a/artifact/client/model.go +++ b/artifact/client/model.go @@ -28,17 +28,17 @@ package client import ( "sort" - "github.com/hashicorp/go-version" - "github.com/nabbar/golib/errors" + hscvrs "github.com/hashicorp/go-version" + liberr "github.com/nabbar/golib/errors" ) type ClientHelper struct { - F func() (releases version.Collection, err errors.Error) + F func() (releases hscvrs.Collection, err liberr.Error) } -func (g *ClientHelper) listReleasesOrderMajor() (releases map[int]version.Collection, err errors.Error) { +func (g *ClientHelper) listReleasesOrderMajor() (releases map[int]hscvrs.Collection, err liberr.Error) { var ( - vers version.Collection + vers hscvrs.Collection ) if vers, err = g.F(); err != nil { @@ -49,7 +49,7 @@ func (g *ClientHelper) listReleasesOrderMajor() (releases map[int]version.Collec s := v.Segments() if releases == nil { - releases = make(map[int]version.Collection) + releases = make(map[int]hscvrs.Collection) } releases[s[0]] = append(releases[s[0]], v) @@ -58,9 +58,9 @@ func (g *ClientHelper) listReleasesOrderMajor() (releases map[int]version.Collec return } -func (g *ClientHelper) ListReleasesOrder() (releases map[int]map[int]version.Collection, err errors.Error) { +func (g *ClientHelper) ListReleasesOrder() (releases map[int]map[int]hscvrs.Collection, err liberr.Error) { var ( - vers map[int]version.Collection + vers map[int]hscvrs.Collection ) if vers, err = g.listReleasesOrderMajor(); err != nil { @@ -72,11 +72,11 @@ func (g *ClientHelper) ListReleasesOrder() (releases map[int]map[int]version.Col s := v.Segments() if releases == nil { - releases = make(map[int]map[int]version.Collection) + releases = make(map[int]map[int]hscvrs.Collection) } if releases[major] == nil || len(releases[major]) == 0 { - releases[major] = make(map[int]version.Collection) + releases[major] = make(map[int]hscvrs.Collection) } releases[major][s[1]] = append(releases[major][s[1]], v) @@ -86,9 +86,9 @@ func (g *ClientHelper) ListReleasesOrder() (releases map[int]map[int]version.Col return } -func (g *ClientHelper) ListReleasesMajor(major int) (releases version.Collection, err errors.Error) { +func (g *ClientHelper) ListReleasesMajor(major int) (releases hscvrs.Collection, err liberr.Error) { var ( - vers map[int]version.Collection + vers map[int]hscvrs.Collection ) if vers, err = g.listReleasesOrderMajor(); err != nil { @@ -106,9 +106,9 @@ func (g *ClientHelper) ListReleasesMajor(major int) (releases version.Collection return } -func (g *ClientHelper) ListReleasesMinor(major, minor int) (releases version.Collection, err errors.Error) { +func (g *ClientHelper) ListReleasesMinor(major, minor int) (releases hscvrs.Collection, err liberr.Error) { var ( - vers map[int]map[int]version.Collection + vers map[int]map[int]hscvrs.Collection ) if vers, err = g.ListReleasesOrder(); err != nil { @@ -130,9 +130,9 @@ func (g *ClientHelper) ListReleasesMinor(major, minor int) (releases version.Col return } -func (g *ClientHelper) GetLatest() (release *version.Version, err errors.Error) { +func (g *ClientHelper) GetLatest() (release *hscvrs.Version, err liberr.Error) { var ( - vers map[int]map[int]version.Collection + vers map[int]map[int]hscvrs.Collection major int minor int ) @@ -156,9 +156,9 @@ func (g *ClientHelper) GetLatest() (release *version.Version, err errors.Error) return g.GetLatestMinor(major, minor) } -func (g *ClientHelper) GetLatestMajor(major int) (release *version.Version, err errors.Error) { +func (g *ClientHelper) GetLatestMajor(major int) (release *hscvrs.Version, err liberr.Error) { var ( - vers map[int]map[int]version.Collection + vers map[int]map[int]hscvrs.Collection minor int ) @@ -179,9 +179,9 @@ func (g *ClientHelper) GetLatestMajor(major int) (release *version.Version, err return g.GetLatestMinor(major, minor) } -func (g *ClientHelper) GetLatestMinor(major, minor int) (release *version.Version, err errors.Error) { +func (g *ClientHelper) GetLatestMinor(major, minor int) (release *hscvrs.Version, err liberr.Error) { var ( - vers version.Collection + vers hscvrs.Collection ) if vers, err = g.ListReleasesMinor(major, minor); err != nil { diff --git a/artifact/github/error.go b/artifact/github/error.go index ea2f4eb..7edbfc7 100644 --- a/artifact/github/error.go +++ b/artifact/github/error.go @@ -27,14 +27,17 @@ package github import ( - err2 "errors" + "errors" + "fmt" - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/errors" + libart "github.com/nabbar/golib/artifact" + liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/artifact/github" + const ( - ErrorParamsEmpty errors.CodeError = iota + artifact.MIN_ARTIFACT_GITHUB + ErrorParamEmpty liberr.CodeError = iota + libart.MinArtifactGithub ErrorURLParse ErrorClientInit ErrorGithubList @@ -49,27 +52,24 @@ const ( ) var ( - isCodeError = false - errResponseCode = err2.New("response status code %s") - errResponseContents = err2.New("response contents is empty") - errResponseBodyEmpty = err2.New("empty body response") - errMisMatchingSize = err2.New("destination size and contentLenght header are not matching") + errResponseCode = errors.New("response status code %s") + errResponseContents = errors.New("response contents is empty") + errResponseBodyEmpty = errors.New("empty body response") + errMisMatchingSize = errors.New("destination size and contentLength header are not matching") ) -func IsCodeError() bool { - return isCodeError -} - func init() { - isCodeError = errors.ExistInMapMessage(ErrorParamsEmpty) - errors.RegisterIdFctMessage(ErrorParamsEmpty, getMessage) + if liberr.ExistInMapMessage(ErrorParamEmpty) { + panic(fmt.Errorf("error code collision with package %s", pkgName)) + } + liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } -func getMessage(code errors.CodeError) (message string) { +func getMessage(code liberr.CodeError) (message string) { switch code { - case errors.UNK_ERROR: - return "" - case ErrorParamsEmpty: + case liberr.UnknownError: + return liberr.NullMessage + case ErrorParamEmpty: return "given parameters is empty" case ErrorURLParse: return "github endpoint seems to be not valid" @@ -95,5 +95,5 @@ func getMessage(code errors.CodeError) (message string) { return "mismatching size between downloaded contents and github http response header" } - return "" + return liberr.NullMessage } diff --git a/artifact/github/github.go b/artifact/github/github.go index 3f85bcd..da25eec 100644 --- a/artifact/github/github.go +++ b/artifact/github/github.go @@ -30,10 +30,10 @@ import ( "net/http" "strings" - "github.com/google/go-github/v33/github" - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/artifact/client" - "github.com/nabbar/golib/errors" + github "github.com/google/go-github/v33/github" + libart "github.com/nabbar/golib/artifact" + artcli "github.com/nabbar/golib/artifact/client" + liberr "github.com/nabbar/golib/errors" ) func getOrgProjectFromRepos(repos string) (owner string, project string) { @@ -45,11 +45,11 @@ func getOrgProjectFromRepos(repos string) (owner string, project string) { return lst[0], lst[1] } -func NewGithub(ctx context.Context, httpcli *http.Client, repos string) (cli artifact.Client, err errors.Error) { +func NewGithub(ctx context.Context, httpcli *http.Client, repos string) (cli libart.Client, err liberr.Error) { o, p := getOrgProjectFromRepos(repos) a := &githubModel{ - ClientHelper: client.ClientHelper{}, + ClientHelper: artcli.ClientHelper{}, c: github.NewClient(httpcli), x: ctx, o: o, @@ -61,11 +61,11 @@ func NewGithub(ctx context.Context, httpcli *http.Client, repos string) (cli art return a, err } -func NewGithubWithTokenOAuth(ctx context.Context, repos string, oauth2client *http.Client) (cli artifact.Client, err errors.Error) { +func NewGithubWithTokenOAuth(ctx context.Context, repos string, oauth2client *http.Client) (cli libart.Client, err liberr.Error) { o, p := getOrgProjectFromRepos(repos) a := &githubModel{ - ClientHelper: client.ClientHelper{}, + ClientHelper: artcli.ClientHelper{}, c: github.NewClient(oauth2client), x: ctx, o: o, diff --git a/artifact/github/model.go b/artifact/github/model.go index ac76e0e..1033214 100644 --- a/artifact/github/model.go +++ b/artifact/github/model.go @@ -29,16 +29,15 @@ import ( "context" "io" "net/http" - "os" "sort" "strings" - "github.com/google/go-github/v33/github" - "github.com/hashicorp/go-version" - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/artifact/client" - "github.com/nabbar/golib/errors" - "github.com/nabbar/golib/ioutils" + github "github.com/google/go-github/v33/github" + hscvrs "github.com/hashicorp/go-version" + libart "github.com/nabbar/golib/artifact" + artcli "github.com/nabbar/golib/artifact/client" + liberr "github.com/nabbar/golib/errors" + libfpg "github.com/nabbar/golib/file/progress" ) const ( @@ -46,7 +45,7 @@ const ( ) type githubModel struct { - client.ClientHelper + artcli.ClientHelper c *github.Client x context.Context @@ -54,7 +53,7 @@ type githubModel struct { p string } -func (g *githubModel) ListReleases() (releases version.Collection, err errors.Error) { +func (g *githubModel) ListReleases() (releases hscvrs.Collection, err liberr.Error) { var ( e error lopt = &github.ListOptions{ @@ -77,9 +76,9 @@ func (g *githubModel) ListReleases() (releases version.Collection, err errors.Er } for _, r := range rels { - v, _ := version.NewVersion(*r.TagName) + v, _ := hscvrs.NewVersion(*r.TagName) - if artifact.ValidatePreRelease(v) { + if libart.ValidatePreRelease(v) { releases = append(releases, v) } } @@ -93,7 +92,7 @@ func (g *githubModel) ListReleases() (releases version.Collection, err errors.Er } } -func (g *githubModel) GetArtifact(containName string, regexName string, release *version.Version) (link string, err errors.Error) { +func (g *githubModel) GetArtifact(containName string, regexName string, release *hscvrs.Version) (link string, err liberr.Error) { var ( rels *github.RepositoryRelease e error @@ -106,7 +105,7 @@ func (g *githubModel) GetArtifact(containName string, regexName string, release for _, a := range rels.Assets { if containName != "" && strings.Contains(*a.Name, containName) { return *a.BrowserDownloadURL, nil - } else if regexName != "" && artifact.CheckRegex(regexName, *a.Name) { + } else if regexName != "" && libart.CheckRegex(regexName, *a.Name) { return *a.BrowserDownloadURL, nil } } @@ -114,14 +113,14 @@ func (g *githubModel) GetArtifact(containName string, regexName string, release return "", ErrorGithubNotFound.Error(nil) } -func (g *githubModel) Download(dst ioutils.FileProgress, containName string, regexName string, release *version.Version) errors.Error { +func (g *githubModel) Download(dst libfpg.Progress, containName string, regexName string, release *hscvrs.Version) liberr.Error { var ( uri string - inf os.FileInfo rsp *github.Response req *http.Request err error - e errors.Error + e liberr.Error + n int64 ) defer func() { @@ -145,11 +144,13 @@ func (g *githubModel) Download(dst ioutils.FileProgress, containName string, reg return ErrorGithubResponse.ErrorParent(errResponseContents) } else if rsp.Body == nil { return ErrorGithubResponse.ErrorParent(errResponseBodyEmpty) - } else if _, err = io.Copy(dst, rsp.Body); err != nil { + } else { + dst.Reset(rsp.ContentLength) + } + + if n, err = io.Copy(dst, rsp.Body); err != nil { return ErrorGithubIOCopy.ErrorParent(err) - } else if inf, err = dst.FileStat(); err != nil { - return ErrorDestinationStat.ErrorParent(err) - } else if inf.Size() != rsp.ContentLength { + } else if n != rsp.ContentLength { return ErrorDestinationSize.ErrorParent(errMisMatchingSize) } diff --git a/artifact/gitlab/error.go b/artifact/gitlab/error.go index a2678d8..cbaa1ca 100644 --- a/artifact/gitlab/error.go +++ b/artifact/gitlab/error.go @@ -27,14 +27,17 @@ package gitlab import ( - err2 "errors" + "errors" + "fmt" - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/errors" + libart "github.com/nabbar/golib/artifact" + liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/artifact/gitlab" + const ( - ErrorParamsEmpty errors.CodeError = iota + artifact.MIN_ARTIFACT_GITLAB + ErrorParamEmpty liberr.CodeError = iota + libart.MinArtifactGitlab ErrorURLParse ErrorClientInit ErrorGitlabList @@ -49,27 +52,24 @@ const ( ) var ( - isCodeError = false - errResponseCode = err2.New("response status code %s") - errResponseContents = err2.New("response contents is empty") - errResponseBodyEmpty = err2.New("empty body response") - errMisMatchingSize = err2.New("destination size and contentLenght header are not matching") + errResponseCode = errors.New("response status code %s") + errResponseContents = errors.New("response contents is empty") + errResponseBodyEmpty = errors.New("empty body response") + errMisMatchingSize = errors.New("destination size and contentLength header are not matching") ) -func IsCodeError() bool { - return isCodeError -} - func init() { - isCodeError = errors.ExistInMapMessage(ErrorParamsEmpty) - errors.RegisterIdFctMessage(ErrorParamsEmpty, getMessage) + if liberr.ExistInMapMessage(ErrorParamEmpty) { + panic(fmt.Errorf("error code collision with package %s", pkgName)) + } + liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } -func getMessage(code errors.CodeError) (message string) { +func getMessage(code liberr.CodeError) (message string) { switch code { - case errors.UNK_ERROR: - return "" - case ErrorParamsEmpty: + case liberr.UnknownError: + return liberr.NullMessage + case ErrorParamEmpty: return "given parameters is empty" case ErrorURLParse: return "gitlab endpoint seems to be not valid" @@ -95,5 +95,5 @@ func getMessage(code errors.CodeError) (message string) { return "mismatching size between downloaded contents and gitlab http response header" } - return "" + return liberr.NullMessage } diff --git a/artifact/gitlab/gitlab.go b/artifact/gitlab/gitlab.go index 905ebe8..81e261a 100644 --- a/artifact/gitlab/gitlab.go +++ b/artifact/gitlab/gitlab.go @@ -31,10 +31,10 @@ import ( "net/url" "strings" - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/artifact/client" - "github.com/nabbar/golib/errors" - "github.com/xanzy/go-gitlab" + libart "github.com/nabbar/golib/artifact" + artcli "github.com/nabbar/golib/artifact/client" + liberr "github.com/nabbar/golib/errors" + gitlab "github.com/xanzy/go-gitlab" ) const ( @@ -42,7 +42,7 @@ const ( GitlabAPIVersion = "/v4" ) -func getGitlbaOptions(baseUrl string, httpcli *http.Client) (opt []gitlab.ClientOptionFunc, err errors.Error) { +func getGitlbaOptions(baseUrl string, httpcli *http.Client) (opt []gitlab.ClientOptionFunc, err liberr.Error) { var ( u *url.URL e error @@ -71,9 +71,9 @@ func getGitlbaOptions(baseUrl string, httpcli *http.Client) (opt []gitlab.Client return } -func newGitlab(ctx context.Context, c *gitlab.Client, projectId int) artifact.Client { +func newGitlab(ctx context.Context, c *gitlab.Client, projectId int) libart.Client { a := &gitlabModel{ - ClientHelper: client.ClientHelper{}, + ClientHelper: artcli.ClientHelper{}, c: c, x: ctx, p: projectId, @@ -84,7 +84,7 @@ func newGitlab(ctx context.Context, c *gitlab.Client, projectId int) artifact.Cl return a } -func NewGitlabAuthUser(ctx context.Context, httpcli *http.Client, user, pass, baseUrl string, projectId int) (cli artifact.Client, err errors.Error) { +func NewGitlabAuthUser(ctx context.Context, httpcli *http.Client, user, pass, baseUrl string, projectId int) (cli libart.Client, err liberr.Error) { var ( o []gitlab.ClientOptionFunc c *gitlab.Client @@ -102,7 +102,7 @@ func NewGitlabAuthUser(ctx context.Context, httpcli *http.Client, user, pass, ba return newGitlab(ctx, c, projectId), err } -func NewGitlabOAuth(ctx context.Context, httpcli *http.Client, oAuthToken, baseUrl string, projectId int) (cli artifact.Client, err errors.Error) { +func NewGitlabOAuth(ctx context.Context, httpcli *http.Client, oAuthToken, baseUrl string, projectId int) (cli libart.Client, err liberr.Error) { var ( o []gitlab.ClientOptionFunc c *gitlab.Client @@ -120,7 +120,7 @@ func NewGitlabOAuth(ctx context.Context, httpcli *http.Client, oAuthToken, baseU return newGitlab(ctx, c, projectId), err } -func NewGitlabPrivateToken(ctx context.Context, httpcli *http.Client, token, baseUrl string, projectId int) (cli artifact.Client, err errors.Error) { +func NewGitlabPrivateToken(ctx context.Context, httpcli *http.Client, token, baseUrl string, projectId int) (cli libart.Client, err liberr.Error) { var ( o []gitlab.ClientOptionFunc c *gitlab.Client diff --git a/artifact/gitlab/model.go b/artifact/gitlab/model.go index c2ef2ed..efe4d0b 100644 --- a/artifact/gitlab/model.go +++ b/artifact/gitlab/model.go @@ -29,17 +29,16 @@ import ( "context" "io" "net/http" - "os" "sort" "strings" - "github.com/hashicorp/go-retryablehttp" - "github.com/hashicorp/go-version" - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/artifact/client" - "github.com/nabbar/golib/errors" - "github.com/nabbar/golib/ioutils" - "github.com/xanzy/go-gitlab" + hschtc "github.com/hashicorp/go-retryablehttp" + hscvrs "github.com/hashicorp/go-version" + libart "github.com/nabbar/golib/artifact" + artcli "github.com/nabbar/golib/artifact/client" + liberr "github.com/nabbar/golib/errors" + libfpg "github.com/nabbar/golib/file/progress" + gitlab "github.com/xanzy/go-gitlab" ) const ( @@ -47,14 +46,14 @@ const ( ) type gitlabModel struct { - client.ClientHelper + artcli.ClientHelper c *gitlab.Client x context.Context p int } -func (g *gitlabModel) ListReleases() (releases version.Collection, err errors.Error) { +func (g *gitlabModel) ListReleases() (releases hscvrs.Collection, err liberr.Error) { var ( e error lopt = &gitlab.ListReleasesOptions{ @@ -76,9 +75,9 @@ func (g *gitlabModel) ListReleases() (releases version.Collection, err errors.Er } for _, r := range rels { - v, _ := version.NewVersion(r.TagName) + v, _ := hscvrs.NewVersion(r.TagName) - if artifact.ValidatePreRelease(v) { + if libart.ValidatePreRelease(v) { releases = append(releases, v) } } @@ -92,7 +91,7 @@ func (g *gitlabModel) ListReleases() (releases version.Collection, err errors.Er } } -func (g *gitlabModel) GetArtifact(containName string, regexName string, release *version.Version) (link string, err errors.Error) { +func (g *gitlabModel) GetArtifact(containName string, regexName string, release *hscvrs.Version) (link string, err liberr.Error) { var ( vers *gitlab.Release e error @@ -105,7 +104,7 @@ func (g *gitlabModel) GetArtifact(containName string, regexName string, release for _, l := range vers.Assets.Links { if containName != "" && strings.Contains(l.Name, containName) { return l.URL, nil - } else if regexName != "" && artifact.CheckRegex(regexName, l.Name) { + } else if regexName != "" && libart.CheckRegex(regexName, l.Name) { return l.URL, nil } } @@ -113,14 +112,14 @@ func (g *gitlabModel) GetArtifact(containName string, regexName string, release return "", ErrorGitlabNotFound.Error(nil) } -func (g *gitlabModel) Download(dst ioutils.FileProgress, containName string, regexName string, release *version.Version) errors.Error { +func (g *gitlabModel) Download(dst libfpg.Progress, containName string, regexName string, release *hscvrs.Version) liberr.Error { var ( uri string - inf os.FileInfo rsp *gitlab.Response - req *retryablehttp.Request + req *hschtc.Request err error - e errors.Error + e liberr.Error + n int64 ) defer func() { @@ -144,11 +143,13 @@ func (g *gitlabModel) Download(dst ioutils.FileProgress, containName string, reg return ErrorGitlabResponse.ErrorParent(errResponseContents) } else if rsp.Body == nil { return ErrorGitlabResponse.ErrorParent(errResponseBodyEmpty) - } else if _, err = io.Copy(dst, rsp.Body); err != nil { + } else { + dst.Reset(rsp.ContentLength) + } + + if n, err = io.Copy(dst, rsp.Body); err != nil { return ErrorGitlabIOCopy.ErrorParent(err) - } else if inf, err = dst.FileStat(); err != nil { - return ErrorDestinationStat.ErrorParent(err) - } else if inf.Size() != rsp.ContentLength { + } else if n != rsp.ContentLength { return ErrorDestinationSize.ErrorParent(errMisMatchingSize) } diff --git a/artifact/jfrog/artifactory.go b/artifact/jfrog/artifactory.go index bd8624a..68cb94e 100644 --- a/artifact/jfrog/artifactory.go +++ b/artifact/jfrog/artifactory.go @@ -30,17 +30,17 @@ import ( "net/http" "net/url" - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/artifact/client" - "github.com/nabbar/golib/errors" + libart "github.com/nabbar/golib/artifact" + artcli "github.com/nabbar/golib/artifact/client" + liberr "github.com/nabbar/golib/errors" ) -func NewArtifactory(ctx context.Context, Do func(req *http.Request) (*http.Response, error), uri, releaseRegex string, releaseGroup int, reposPath ...string) (artifact.Client, errors.Error) { +func NewArtifactory(ctx context.Context, Do func(req *http.Request) (*http.Response, error), uri, releaseRegex string, releaseGroup int, reposPath ...string) (libart.Client, liberr.Error) { if u, e := url.Parse(uri); e != nil { return nil, ErrorURLParse.ErrorParent(e) } else { a := &artifactoryModel{ - ClientHelper: client.ClientHelper{}, + ClientHelper: artcli.ClientHelper{}, Do: Do, ctx: ctx, endpoint: u, diff --git a/artifact/jfrog/error.go b/artifact/jfrog/error.go index b0dc395..7a2eb05 100644 --- a/artifact/jfrog/error.go +++ b/artifact/jfrog/error.go @@ -27,12 +27,17 @@ package jfrog import ( - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/errors" + "errors" + "fmt" + + libart "github.com/nabbar/golib/artifact" + liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/artifact/jfrog" + const ( - ErrorParamsEmpty errors.CodeError = iota + artifact.MIN_ARTIFACT_JFORG + ErrorParamEmpty liberr.CodeError = iota + libart.MinArtifactJfrog ErrorURLParse ErrorRequestInit ErrorRequestDo @@ -41,24 +46,25 @@ const ( ErrorRequestResponseBodyDecode ErrorArtifactoryNotFound ErrorArtifactoryDownload + ErrorDestinationSize ) -var isCodeError = false - -func IsCodeError() bool { - return isCodeError -} +var ( + errMisMatchingSize = errors.New("destination size and contentLength header are not matching") +) func init() { - isCodeError = errors.ExistInMapMessage(ErrorParamsEmpty) - errors.RegisterIdFctMessage(ErrorParamsEmpty, getMessage) + if liberr.ExistInMapMessage(ErrorParamEmpty) { + panic(fmt.Errorf("error code collision with package %s", pkgName)) + } + liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } -func getMessage(code errors.CodeError) (message string) { +func getMessage(code liberr.CodeError) (message string) { switch code { - case errors.UNK_ERROR: - return "" - case ErrorParamsEmpty: + case liberr.UnknownError: + return liberr.NullMessage + case ErrorParamEmpty: return "given parameters is empty" case ErrorURLParse: return "endpoint of JFrog Artifactory seems to be not valid" @@ -78,5 +84,5 @@ func getMessage(code errors.CodeError) (message string) { return "error on downloading artifact" } - return "" + return liberr.NullMessage } diff --git a/artifact/jfrog/model.go b/artifact/jfrog/model.go index 22182ca..8c87209 100644 --- a/artifact/jfrog/model.go +++ b/artifact/jfrog/model.go @@ -29,6 +29,7 @@ import ( "context" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" "net/url" @@ -38,11 +39,11 @@ import ( "strings" "time" - "github.com/hashicorp/go-version" + hscvrs "github.com/hashicorp/go-version" libart "github.com/nabbar/golib/artifact" artcli "github.com/nabbar/golib/artifact/client" liberr "github.com/nabbar/golib/errors" - libiot "github.com/nabbar/golib/ioutils" + libfpg "github.com/nabbar/golib/file/progress" ) type artifactoryModel struct { @@ -187,12 +188,12 @@ func (a *artifactoryModel) getStorageList() (sto []ResponseStorage, err liberr.E if a.regex == "" { //nolint #goerr113 - return nil, ErrorParamsEmpty.ErrorParent(fmt.Errorf("regex is empty: %s", a.regex)) + return nil, ErrorParamEmpty.ErrorParent(fmt.Errorf("regex is empty: %s", a.regex)) } if a.group < 1 { //nolint #goerr113 - return nil, ErrorParamsEmpty.ErrorParent(fmt.Errorf("group extracted from regex is empty: %s - %v", a.regex, a.group)) + return nil, ErrorParamEmpty.ErrorParent(fmt.Errorf("group extracted from regex is empty: %s - %v", a.regex, a.group)) } if err = a.request("", &lst); err != nil { @@ -231,7 +232,7 @@ func (a *artifactoryModel) getStorageList() (sto []ResponseStorage, err liberr.E return sto, nil } -func (a *artifactoryModel) releasesAppendNotExist(releases version.Collection, vers *version.Version) version.Collection { +func (a *artifactoryModel) releasesAppendNotExist(releases hscvrs.Collection, vers *hscvrs.Version) hscvrs.Collection { for _, k := range releases { if k.Equal(vers) { return releases @@ -241,7 +242,7 @@ func (a *artifactoryModel) releasesAppendNotExist(releases version.Collection, v return append(releases, vers) } -func (a *artifactoryModel) ListReleases() (releases version.Collection, err liberr.Error) { +func (a *artifactoryModel) ListReleases() (releases hscvrs.Collection, err liberr.Error) { var ( reg = regexp.MustCompile(a.regex) sto []ResponseStorage @@ -258,7 +259,7 @@ func (a *artifactoryModel) ListReleases() (releases version.Collection, err libe continue } - if v, e := version.NewVersion(grp[a.group]); e != nil { + if v, e := hscvrs.NewVersion(grp[a.group]); e != nil { continue } else if !libart.ValidatePreRelease(v) { continue @@ -270,7 +271,7 @@ func (a *artifactoryModel) ListReleases() (releases version.Collection, err libe return releases, nil } -func (a *artifactoryModel) getArtifact(containName string, regexName string, release *version.Version) (art *ResponseStorage, err liberr.Error) { +func (a *artifactoryModel) getArtifact(containName string, regexName string, release *hscvrs.Version) (art *ResponseStorage, err liberr.Error) { var ( reg = regexp.MustCompile(a.regex) rg2 *regexp.Regexp @@ -293,7 +294,7 @@ func (a *artifactoryModel) getArtifact(containName string, regexName string, rel continue } - if v, e := version.NewVersion(grp[a.group]); e != nil { + if v, e := hscvrs.NewVersion(grp[a.group]); e != nil { continue } else if !libart.ValidatePreRelease(v) { continue @@ -311,7 +312,7 @@ func (a *artifactoryModel) getArtifact(containName string, regexName string, rel return nil, ErrorArtifactoryNotFound.Error(nil) } -func (a *artifactoryModel) GetArtifact(containName string, regexName string, release *version.Version) (link string, err liberr.Error) { +func (a *artifactoryModel) GetArtifact(containName string, regexName string, release *hscvrs.Version) (link string, err liberr.Error) { if art, err := a.getArtifact(containName, regexName, release); err != nil { return "", err } else { @@ -319,9 +320,10 @@ func (a *artifactoryModel) GetArtifact(containName string, regexName string, rel } } -func (a *artifactoryModel) Download(dst libiot.FileProgress, containName string, regexName string, release *version.Version) liberr.Error { +func (a *artifactoryModel) Download(dst libfpg.Progress, containName string, regexName string, release *hscvrs.Version) liberr.Error { var ( e error + n int64 art *ResponseStorage err liberr.Error @@ -341,10 +343,10 @@ func (a *artifactoryModel) Download(dst libiot.FileProgress, containName string, if art, err = a.getArtifact(containName, regexName, release); err != nil { return err + } else { + dst.Reset(art.size) } - dst.ResetMax(art.size) - if req, e = http.NewRequestWithContext(a.ctx, http.MethodGet, art.DownloadUri, nil); e != nil { return ErrorRequestInit.ErrorParent(e) } else if rsp, e = a.Do(req); e != nil { @@ -355,8 +357,12 @@ func (a *artifactoryModel) Download(dst libiot.FileProgress, containName string, } else if rsp.Body == nil { //nolint #goerr113 return ErrorRequestResponseBodyEmpty.ErrorParent(fmt.Errorf("status: %v", rsp.Status)) - } else if _, e := dst.ReadFrom(rsp.Body); e != nil { + } else if n, e = io.Copy(dst, rsp.Body); e != nil { return ErrorArtifactoryDownload.ErrorParent(e) + } else if n != art.size { + return ErrorDestinationSize.ErrorParent(errMisMatchingSize) + } else if n != rsp.ContentLength { + return ErrorDestinationSize.ErrorParent(errMisMatchingSize) } return nil diff --git a/artifact/s3aws/error.go b/artifact/s3aws/error.go index 47c9a31..765699f 100644 --- a/artifact/s3aws/error.go +++ b/artifact/s3aws/error.go @@ -29,12 +29,14 @@ package s3aws import ( "fmt" - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/errors" + libart "github.com/nabbar/golib/artifact" + liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/artifact/s3aws" + const ( - ErrorParamsEmpty errors.CodeError = iota + artifact.MIN_ARTIFACT_S3AWS + ErrorParamEmpty liberr.CodeError = iota + libart.MinArtifactS3AWS ErrorClientInit ErrorS3AWSRegex ErrorS3AWSFind @@ -45,7 +47,6 @@ const ( ) var ( - isCodeError = false errRegexGroup = "regex '%s' has only '%d' group extracted and given group to use '%d'" errVersion = "error with version '%s'" errVersRequest = "version requested '%s'" @@ -59,20 +60,18 @@ func getError(code string, args ...interface{}) error { return fmt.Errorf(code, args...) } -func IsCodeError() bool { - return isCodeError -} - func init() { - isCodeError = errors.ExistInMapMessage(ErrorParamsEmpty) - errors.RegisterIdFctMessage(ErrorParamsEmpty, getMessage) + if liberr.ExistInMapMessage(ErrorParamEmpty) { + panic(fmt.Errorf("error code collision with package %s", pkgName)) + } + liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } -func getMessage(code errors.CodeError) (message string) { +func getMessage(code liberr.CodeError) (message string) { switch code { - case errors.UNK_ERROR: - return "" - case ErrorParamsEmpty: + case liberr.UnknownError: + return liberr.NullMessage + case ErrorParamEmpty: return "given parameters is empty" case ErrorClientInit: return "initialization of gitlab client failed" @@ -90,5 +89,5 @@ func getMessage(code errors.CodeError) (message string) { return "return io reader is empty" } - return "" + return liberr.NullMessage } diff --git a/artifact/s3aws/model.go b/artifact/s3aws/model.go index c449b05..732fa1e 100644 --- a/artifact/s3aws/model.go +++ b/artifact/s3aws/model.go @@ -28,37 +28,36 @@ package s3aws import ( "context" "io" - "os" "regexp" "strings" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/hashicorp/go-version" - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/artifact/client" - "github.com/nabbar/golib/aws" - "github.com/nabbar/golib/errors" - "github.com/nabbar/golib/ioutils" + sdksss "github.com/aws/aws-sdk-go-v2/service/s3" + hscvrs "github.com/hashicorp/go-version" + libart "github.com/nabbar/golib/artifact" + artcli "github.com/nabbar/golib/artifact/client" + libaws "github.com/nabbar/golib/aws" + liberr "github.com/nabbar/golib/errors" + libfpg "github.com/nabbar/golib/file/progress" ) type s3awsModel struct { - client.ClientHelper + artcli.ClientHelper - c aws.AWS + c libaws.AWS x context.Context regex string group int } -func (s *s3awsModel) ListReleases() (releases version.Collection, err errors.Error) { +func (s *s3awsModel) ListReleases() (releases hscvrs.Collection, err liberr.Error) { var ( - e errors.Error + e liberr.Error r *regexp.Regexp l []string ) if s.regex == "" { - return nil, ErrorParamsEmpty.Error(nil) + return nil, ErrorParamEmpty.Error(nil) } if l, e = s.c.Object().Find(s.regex); e != nil { @@ -74,9 +73,9 @@ func (s *s3awsModel) ListReleases() (releases version.Collection, err errors.Err return nil, ErrorS3AWSRegex.ErrorParent(getError(errRegexGroup, s.regex, len(grp), s.group)) } - if v, e := version.NewVersion(grp[s.group]); e != nil { + if v, e := hscvrs.NewVersion(grp[s.group]); e != nil { continue - } else if !artifact.ValidatePreRelease(v) { + } else if !libart.ValidatePreRelease(v) { continue } else { var found bool @@ -95,19 +94,21 @@ func (s *s3awsModel) ListReleases() (releases version.Collection, err errors.Err return releases, nil } -func (s *s3awsModel) GetArtifact(containName string, regexName string, release *version.Version) (link string, err errors.Error) { +func (s *s3awsModel) GetArtifact(containName string, regexName string, release *hscvrs.Version) (link string, err liberr.Error) { var ( - e errors.Error - r *regexp.Regexp + e error + k bool l []string + r *regexp.Regexp + v *hscvrs.Version ) if s.regex == "" { - return "", ErrorParamsEmpty.Error(e) + return "", ErrorParamEmpty.Error(nil) } - if l, e = s.c.Object().Find(s.regex); e != nil { - return "", ErrorS3AWSFind.Error(e) + if l, err = s.c.Object().Find(s.regex); err != nil { + return "", ErrorS3AWSFind.Error(err) } r = regexp.MustCompile(s.regex) @@ -119,7 +120,7 @@ func (s *s3awsModel) GetArtifact(containName string, regexName string, release * return "", ErrorS3AWSRegex.ErrorParent(getError(errRegexGroup, s.regex, len(grp), s.group)) } - if v, e := version.NewVersion(grp[s.group]); e != nil { + if v, e = hscvrs.NewVersion(grp[s.group]); e != nil { return "", ErrorS3AWSNewVers.ErrorParent(getError(errVersion, grp[s.group]), e) } else if v.Equal(release) { uri := s.c.Config().GetEndpoint() @@ -131,7 +132,7 @@ func (s *s3awsModel) GetArtifact(containName string, regexName string, release * } if regexName != "" { - if ok, e := regexp.MatchString(regexName, o); e == nil && ok { + if k, e = regexp.MatchString(regexName, o); e == nil && k { return uri.String(), nil } } @@ -145,19 +146,23 @@ func (s *s3awsModel) GetArtifact(containName string, regexName string, release * return "", ErrorS3AWSNotFound.ErrorParent(getError(errVersRequest, release.String())) } -func (s *s3awsModel) Download(dst ioutils.FileProgress, containName string, regexName string, release *version.Version) errors.Error { +func (s *s3awsModel) Download(dst libfpg.Progress, containName string, regexName string, release *hscvrs.Version) liberr.Error { var ( - e errors.Error + e error r *regexp.Regexp l []string + v *hscvrs.Version + k bool + + err liberr.Error ) if s.regex == "" { - return ErrorParamsEmpty.Error(e) + return ErrorParamEmpty.Error(nil) } - if l, e = s.c.Object().Find(s.regex); e != nil { - return ErrorS3AWSFind.Error(e) + if l, err = s.c.Object().Find(s.regex); e != nil { + return ErrorS3AWSFind.Error(err) } r = regexp.MustCompile(s.regex) @@ -169,7 +174,7 @@ func (s *s3awsModel) Download(dst ioutils.FileProgress, containName string, rege return ErrorS3AWSRegex.ErrorParent(getError(errRegexGroup, s.regex, len(grp), s.group)) } - if v, e := version.NewVersion(grp[s.group]); e != nil { + if v, e = hscvrs.NewVersion(grp[s.group]); e != nil { return ErrorS3AWSNewVers.ErrorParent(getError(errVersion, grp[s.group]), e) } else if v.Equal(release) { if containName != "" && strings.Contains(o, containName) { @@ -177,12 +182,8 @@ func (s *s3awsModel) Download(dst ioutils.FileProgress, containName string, rege } if regexName != "" { - if ok, e := regexp.MatchString(regexName, o); e == nil && ok { + if k, e = regexp.MatchString(regexName, o); e == nil && k { return s.downloadObject(dst, o) - } else if e != nil { - println(e) - } else { - println("regex " + regexName + " => KO - " + o) } } @@ -195,12 +196,14 @@ func (s *s3awsModel) Download(dst ioutils.FileProgress, containName string, rege return ErrorS3AWSNotFound.ErrorParent(getError(errVersRequest, release.String())) } -func (s *s3awsModel) downloadObject(dst ioutils.FileProgress, object string) errors.Error { +func (s *s3awsModel) downloadObject(dst libfpg.Progress, object string) liberr.Error { var ( - r *s3.GetObjectOutput - e errors.Error - i os.FileInfo + r *sdksss.GetObjectOutput + e error j int64 + n int64 + + err liberr.Error ) defer func() { @@ -209,29 +212,25 @@ func (s *s3awsModel) downloadObject(dst ioutils.FileProgress, object string) err } }() - if j, e = s.c.Object().Size(object); e != nil { - err := ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object)) - err.AddParentError(e) + if j, err = s.c.Object().Size(object); err != nil { + err = ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object)) + err.AddParentError(err) return err } else if j < 1 { return ErrorS3AWSDownloadError.ErrorParent(getError(errObjectEmpty, object)) + } else { + dst.Reset(j) } - dst.ResetMax(j) - - if r, e = s.c.Object().Get(object); e != nil { - err := ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object)) - err.AddParentError(e) + if r, err = s.c.Object().Get(object); err != nil { + err = ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object)) + err.AddParentError(err) return err } else if r.Body == nil { return ErrorS3AWSIOReaderError.ErrorParent(getError(errObject, object)) - } else if _, err := io.Copy(dst, r.Body); err != nil { - return ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object), err) - } else if i, e = dst.FileStat(); e != nil { - err := ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object)) - err.AddParentError(e) - return err - } else if i.Size() != j { + } else if n, e = io.Copy(dst, r.Body); e != nil { + return ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object), e) + } else if n != j { return ErrorS3AWSDownloadError.ErrorParent(getError(errObjectSize, object)) } diff --git a/artifact/s3aws/s3aws.go b/artifact/s3aws/s3aws.go index 4a8b3e9..45876eb 100644 --- a/artifact/s3aws/s3aws.go +++ b/artifact/s3aws/s3aws.go @@ -29,19 +29,19 @@ import ( "context" "net/http" - "github.com/nabbar/golib/artifact" - "github.com/nabbar/golib/artifact/client" - "github.com/nabbar/golib/aws" - "github.com/nabbar/golib/errors" + libart "github.com/nabbar/golib/artifact" + artcli "github.com/nabbar/golib/artifact/client" + libaws "github.com/nabbar/golib/aws" + liberr "github.com/nabbar/golib/errors" ) -func NewS3AWS(ctx context.Context, cfg aws.Config, httpcli *http.Client, forceModePath bool, releaseRegex string, releaseGroup int) (cli artifact.Client, err errors.Error) { +func NewS3AWS(ctx context.Context, cfg libaws.Config, httpcli *http.Client, forceModePath bool, releaseRegex string, releaseGroup int) (cli libart.Client, err liberr.Error) { var ( - c aws.AWS - e errors.Error + c libaws.AWS + e liberr.Error ) - if c, e = aws.New(ctx, cfg, httpcli); e != nil { + if c, e = libaws.New(ctx, cfg, httpcli); e != nil { return nil, ErrorClientInit.Error(e) } @@ -56,7 +56,7 @@ func NewS3AWS(ctx context.Context, cfg aws.Config, httpcli *http.Client, forceMo } o := &s3awsModel{ - ClientHelper: client.ClientHelper{}, + ClientHelper: artcli.ClientHelper{}, c: c, x: ctx, regex: releaseRegex, diff --git a/aws/aws_suite_test.go b/aws/aws_suite_test.go index f7a528a..32d536f 100644 --- a/aws/aws_suite_test.go +++ b/aws/aws_suite_test.go @@ -149,7 +149,7 @@ func loadConfig() error { return err } - if err := cfg.Validate(); err != nil { + if err = cfg.Validate(); err != nil { return err } diff --git a/aws/multipart/model.go b/aws/multipart/model.go index 06d7b70..b69901e 100644 --- a/aws/multipart/model.go +++ b/aws/multipart/model.go @@ -27,14 +27,13 @@ package multipart import ( "context" - "fmt" "path/filepath" "sync" sdksss "github.com/aws/aws-sdk-go-v2/service/s3" sdktyp "github.com/aws/aws-sdk-go-v2/service/s3/types" libctx "github.com/nabbar/golib/context" - libiot "github.com/nabbar/golib/ioutils" + libfpg "github.com/nabbar/golib/file/progress" libsiz "github.com/nabbar/golib/size" ) @@ -48,7 +47,7 @@ type mpu struct { o string // object name n int32 // part counter l []sdktyp.CompletedPart // slice of sent part to prepare complete MPU - w libiot.FileProgress // working file or temporary file + w libfpg.Progress // working file or temporary file // trigger function fc func(nPart int, obj string, e error) // on complete @@ -156,7 +155,7 @@ func (m *mpu) RegisterWorkingFile(file string, truncate bool) error { m.w = nil } - m.w, e = libiot.NewFileProgressPathWrite(filepath.Clean(file), true, truncate, 0600) + m.w, e = libfpg.Create(filepath.Clean(file)) if e != nil { return e @@ -167,7 +166,7 @@ func (m *mpu) RegisterWorkingFile(file string, truncate bool) error { return nil } -func (m *mpu) getWorkingFile() (libiot.FileProgress, error) { +func (m *mpu) getWorkingFile() (libfpg.Progress, error) { if m == nil { return nil, ErrInvalidInstance } @@ -201,7 +200,11 @@ func (m *mpu) setTempWorkingFile() error { defer m.m.Unlock() var e error - m.w, e = libiot.NewFileProgressTemp() + m.w, e = libfpg.Temp("") + if e != nil { + _ = m.w.CloseDelete() + } + return e } @@ -219,17 +222,10 @@ func (m *mpu) closeWorkingFile() error { var e error - e = m.w.Truncate(0) - - if er := m.w.Close(); er != nil { - if e != nil { - e = fmt.Errorf("%v, %v", e, er) - } else { - e = er - } - } - + _ = m.w.Truncate(0) + e = m.w.CloseDelete() m.w = nil + return e } diff --git a/aws/multipart/part.go b/aws/multipart/part.go index d126cea..320cacd 100644 --- a/aws/multipart/part.go +++ b/aws/multipart/part.go @@ -30,6 +30,7 @@ import ( //nolint #nosec "crypto/md5" "encoding/base64" + "errors" "fmt" "io" "strings" @@ -37,7 +38,7 @@ import ( sdkaws "github.com/aws/aws-sdk-go-v2/aws" sdksss "github.com/aws/aws-sdk-go-v2/service/s3" sdktyp "github.com/aws/aws-sdk-go-v2/service/s3/types" - libiot "github.com/nabbar/golib/ioutils" + libfpg "github.com/nabbar/golib/file/progress" libsiz "github.com/nabbar/golib/size" ) @@ -109,41 +110,46 @@ func (m *mpu) AddPart(r io.Reader) (n int64, e error) { var ( cli *sdksss.Client res *sdksss.UploadPartOutput - tmp libiot.FileProgress + tmp libfpg.Progress ctx = m.getContext() obj = m.getObject() bck = m.getBucket() mid = m.getMultipartID() + hss string /* #nosec */ //nolint #nosec hsh = md5.New() ) + defer func() { + if tmp != nil { + _ = tmp.CloseDelete() + } + }() + if cli = m.getClient(); cli == nil { return 0, ErrInvalidClient - } else if tmp, e = libiot.NewFileProgressTemp(); e != nil { + } else if tmp, e = libfpg.Temp(""); e != nil { return 0, e } else if tmp == nil { return 0, ErrInvalidTMPFile - } else { - defer func() { - if tmp != nil { - _ = tmp.Close() - } - }() } - if n, e = io.Copy(tmp, r); e != nil || n < 1 { + if n, e = io.Copy(tmp, r); e != nil && !errors.Is(e, io.EOF) { + return n, e + } else if n < 1 { return n, e } else if _, e = tmp.Seek(0, io.SeekStart); e != nil { return 0, e - } else if _, e = tmp.WriteTo(hsh); e != nil { + } else if _, e = tmp.WriteTo(hsh); e != nil && !errors.Is(e, io.EOF) { return 0, e } else if _, e = tmp.Seek(0, io.SeekStart); e != nil { return 0, e } + hss = base64.StdEncoding.EncodeToString(hsh.Sum(nil)) + res, e = cli.UploadPart(ctx, &sdksss.UploadPartInput{ Bucket: sdkaws.String(bck), Key: sdkaws.String(obj), @@ -152,7 +158,7 @@ func (m *mpu) AddPart(r io.Reader) (n int64, e error) { ContentLength: n, Body: tmp, RequestPayer: sdktyp.RequestPayerRequester, - ContentMD5: sdkaws.String(base64.StdEncoding.EncodeToString(hsh.Sum(nil))), + ContentMD5: sdkaws.String(hss), }) if e != nil { @@ -172,7 +178,7 @@ func (m *mpu) AddPart(r io.Reader) (n int64, e error) { func (m *mpu) AddToPart(p []byte) (n int, e error) { var ( - tmp libiot.FileProgress + tmp libfpg.Progress ) if tmp, e = m.getWorkingFile(); e != nil { @@ -191,7 +197,7 @@ func (m *mpu) AddToPart(p []byte) (n int, e error) { if _, e = tmp.Seek(0, io.SeekStart); e != nil { return n, e - } else if s, e = tmp.SizeToEOF(); e != nil { + } else if s, e = tmp.SizeEOF(); e != nil { return n, e } else if _, e = tmp.Seek(0, io.SeekEnd); e != nil { return n, e @@ -234,7 +240,7 @@ func (m *mpu) CurrentSizePart() int64 { var ( e error s int64 - tmp libiot.FileProgress + tmp libfpg.Progress ) if tmp, e = m.getWorkingFile(); e != nil { @@ -244,7 +250,7 @@ func (m *mpu) CurrentSizePart() int64 { } else if _, e = tmp.Seek(0, io.SeekStart); e != nil { return 0 } else { - s, e = tmp.SizeToEOF() + s, e = tmp.SizeEOF() _, _ = tmp.Seek(0, io.SeekEnd) return s } @@ -255,7 +261,7 @@ func (m *mpu) CheckSend(force, close bool) error { err error siz int64 prt = m.getPartSize() - tmp libiot.FileProgress + tmp libfpg.Progress ) if tmp, err = m.getWorkingFile(); err != nil { @@ -264,7 +270,7 @@ func (m *mpu) CheckSend(force, close bool) error { return ErrInvalidTMPFile } else if _, err = tmp.Seek(0, io.SeekStart); err != nil { return err - } else if siz, err = tmp.SizeToEOF(); err != nil { + } else if siz, err = tmp.SizeEOF(); err != nil { return err } else if siz < prt.Int64() && !force { return nil diff --git a/aws/multipart/stop.go b/aws/multipart/stop.go index 7eb7694..d94664b 100644 --- a/aws/multipart/stop.go +++ b/aws/multipart/stop.go @@ -31,7 +31,7 @@ import ( sdkaws "github.com/aws/aws-sdk-go-v2/aws" sdksss "github.com/aws/aws-sdk-go-v2/service/s3" sdktyp "github.com/aws/aws-sdk-go-v2/service/s3/types" - libiot "github.com/nabbar/golib/ioutils" + libfpg "github.com/nabbar/golib/file/progress" ) func (m *mpu) StopMPU(abort bool) error { @@ -44,6 +44,17 @@ func (m *mpu) StopMPU(abort bool) error { lst = m.getPartList() ) + defer func() { + m.m.Lock() + if m.w != nil { + if i, e := m.w.Stat(); e == nil && i.Size() < 1 { + _ = m.w.CloseDelete() + m.w = nil + } + } + m.m.Unlock() + }() + if !abort { if err = m.CheckSend(true, true); err != nil { return err @@ -90,7 +101,7 @@ func (m *mpu) SendObject() error { err error cli *sdksss.Client res *sdksss.PutObjectOutput - tmp libiot.FileProgress + tmp libfpg.Progress ctx = m.getContext() obj = m.getObject() diff --git a/errors/modules.go b/errors/modules.go index 9887d27..9515c5c 100644 --- a/errors/modules.go +++ b/errors/modules.go @@ -26,50 +26,59 @@ package errors +const baseSub = 10 +const baseInc = baseSub * baseSub + const ( - MinPkgArchive = 100 - MinPkgArtifact = 200 - MinPkgCertificate = 300 - MinPkgCluster = 400 - MinPkgConfig = 500 - MinPkgConsole = 800 - MinPkgCrypt = 900 - MinPkgDatabaseGorm = 1000 - MinPkgDatabaseKVDrv = 1010 - MinPkgDatabaseKVMap = 1020 - MinPkgDatabaseKVTbl = 1030 - MinPkgDatabaseKVItm = 1040 - MinPkgFTPClient = 1100 - MinPkgHttpCli = 1200 - MinPkgHttpServer = 1300 - MinPkgHttpServerPool = 1320 - MinPkgIOUtils = 1400 - MinPkgLDAP = 1500 - MinPkgLogger = 1600 - MinPkgMail = 1700 - MinPkgMailer = 1800 - MinPkgMailPooler = 1900 - MinPkgMonitor = 2000 - MinPkgMonitorCfg = 2020 - MinPkgMonitorPool = 2100 - MinPkgNetwork = 2200 - MinPkgNats = 2300 - MinPkgNutsDB = 2400 - MinPkgOAuth = 2500 - MinPkgAws = 2600 - MinPkgRequest = 2700 - MinPkgRouter = 2800 - MinPkgSemaphore = 2900 - MinPkgSMTP = 3000 - MinPkgSMTPConfig = 3050 - MinPkgStatic = 3100 - MinPkgStatus = 3200 - MinPkgSocket = 3300 - MinPkgVersion = 3400 - MinPkgViper = 3500 + MinPkgArchive = baseInc + iota + MinPkgArtifact = baseInc + MinPkgArchive + MinPkgCertificate = baseInc + MinPkgArtifact + MinPkgCluster = baseInc + MinPkgCertificate + MinPkgConfig = baseInc + MinPkgCluster + MinPkgConsole = baseInc + MinPkgConfig + MinPkgCrypt = baseInc + MinPkgConsole - MinAvailable = 4000 + MinPkgDatabaseGorm = baseInc + MinPkgCrypt + MinPkgDatabaseKVDrv = baseSub + MinPkgDatabaseGorm + MinPkgDatabaseKVMap = baseSub + MinPkgDatabaseKVDrv + MinPkgDatabaseKVTbl = baseSub + MinPkgDatabaseKVMap + MinPkgDatabaseKVItm = baseSub + MinPkgDatabaseKVTbl - // MIN_AVAILABLE @Deprecated use MinAvailable constant - MIN_AVAILABLE = MinAvailable + MinPkgFileProgress = baseInc + MinPkgDatabaseGorm + MinPkgFTPClient = baseInc + MinPkgFileProgress + MinPkgHttpCli = baseInc + MinPkgFTPClient + + MinPkgHttpServer = baseInc + MinPkgHttpCli + MinPkgHttpServerPool = baseSub + MinPkgHttpServer + + MinPkgIOUtils = baseInc + MinPkgHttpServer + MinPkgLDAP = baseInc + MinPkgIOUtils + MinPkgLogger = baseInc + MinPkgLDAP + MinPkgMail = baseInc + MinPkgLogger + MinPkgMailer = baseInc + MinPkgMail + MinPkgMailPooler = baseInc + MinPkgMailer + + MinPkgMonitor = baseInc + MinPkgMailPooler + MinPkgMonitorCfg = baseSub + MinPkgMonitor + MinPkgMonitorPool = baseSub + MinPkgMonitorCfg + + MinPkgNetwork = baseInc + MinPkgMonitor + MinPkgNats = baseInc + MinPkgNetwork + MinPkgNutsDB = baseInc + MinPkgNats + MinPkgOAuth = baseInc + MinPkgNutsDB + MinPkgAws = baseInc + MinPkgOAuth + MinPkgRequest = baseInc + MinPkgAws + MinPkgRouter = baseInc + MinPkgRequest + MinPkgSemaphore = baseInc + MinPkgRouter + + MinPkgSMTP = baseInc + MinPkgSemaphore + MinPkgSMTPConfig = baseInc + MinPkgSMTP + + MinPkgStatic = baseInc + MinPkgSMTPConfig + MinPkgStatus = baseInc + MinPkgStatic + MinPkgSocket = baseInc + MinPkgStatus + MinPkgVersion = baseInc + MinPkgSocket + MinPkgViper = baseInc + MinPkgVersion + + MinAvailable = baseInc + MinPkgViper ) diff --git a/file/progress/errors.go b/file/progress/errors.go new file mode 100644 index 0000000..57c1452 --- /dev/null +++ b/file/progress/errors.go @@ -0,0 +1,87 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package progress + +import ( + "fmt" + + liberr "github.com/nabbar/golib/errors" +) + +const pkgName = "golib/file/progress" +const ( + ErrorParamEmpty liberr.CodeError = iota + liberr.MinPkgFileProgress + ErrorSyscallRLimitGet + ErrorSyscallRLimitSet + ErrorIOFileStat + ErrorIOFileSeek + ErrorIOFileTruncate + ErrorIOFileSync + ErrorIOFileOpen + ErrorIOFileTempNew + ErrorIOFileTempClose + ErrorIOFileTempRemove + ErrorNilPointer +) + +func init() { + if liberr.ExistInMapMessage(ErrorParamEmpty) { + panic(fmt.Errorf("error code collision with package %s", pkgName)) + } + liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) +} + +func getMessage(code liberr.CodeError) (message string) { + switch code { + case ErrorParamEmpty: + return "given parameters is empty" + case ErrorSyscallRLimitGet: + return "error on retrieve value in syscall rlimit" + case ErrorSyscallRLimitSet: + return "error on changing value in syscall rlimit" + case ErrorIOFileStat: + return "error occur while trying to get stat of file" + case ErrorIOFileSeek: + return "error occur while trying seek into file" + case ErrorIOFileTruncate: + return "error occur while trying truncate file" + case ErrorIOFileSync: + return "error occur while trying to sync file" + case ErrorIOFileOpen: + return "error occur while trying to open file" + case ErrorIOFileTempNew: + return "error occur while trying to create new temporary file" + case ErrorIOFileTempClose: + return "closing temporary file occurs error" + case ErrorIOFileTempRemove: + return "error occurs on removing temporary file" + case ErrorNilPointer: + return "cannot call function for a nil pointer" + } + + return liberr.NullMessage +} diff --git a/file/progress/interface.go b/file/progress/interface.go new file mode 100644 index 0000000..2e7edca --- /dev/null +++ b/file/progress/interface.go @@ -0,0 +1,155 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package progress + +import ( + "io" + "os" + "sync/atomic" +) + +const DefaultBuffSize = 32 * 1024 // see io.copyBuffer + +type FctIncrement func(size int64) +type FctReset func(size, current int64) +type FctEOF func() + +type GenericIO interface { + io.ReadCloser + io.ReadSeeker + io.ReadWriteCloser + io.ReadWriteSeeker + io.WriteCloser + io.WriteSeeker + io.Reader + io.ReaderFrom + io.ReaderAt + io.Writer + io.WriterAt + io.WriterTo + io.Seeker + io.StringWriter + io.Closer + io.ByteReader + io.ByteWriter +} + +type File interface { + CloseDelete() error + + Path() string + Stat() (os.FileInfo, error) + + SizeBOF() (size int64, err error) + SizeEOF() (size int64, err error) + + Truncate(size int64) error + Sync() error +} + +type Progress interface { + GenericIO + File + + RegisterFctIncrement(fct FctIncrement) + RegisterFctReset(fct FctReset) + RegisterFctEOF(fct FctEOF) + SetBufferSize(size int32) + SetRegisterProgress(f Progress) + + Reset(max int64) +} + +func New(name string, flags int, perm os.FileMode) (Progress, error) { + if f, e := os.OpenFile(name, flags, perm); e != nil { + return nil, e + } else { + return &progress{ + fos: f, + b: new(atomic.Int32), + fi: new(atomic.Value), + fe: new(atomic.Value), + fr: new(atomic.Value), + }, nil + } +} + +func Unique(basePath, pattern string) (Progress, error) { + if f, e := os.CreateTemp(basePath, pattern); e != nil { + return nil, e + } else { + return &progress{ + fos: f, + b: new(atomic.Int32), + fi: new(atomic.Value), + fe: new(atomic.Value), + fr: new(atomic.Value), + }, nil + } +} + +func Temp(pattern string) (Progress, error) { + if f, e := os.CreateTemp("", pattern); e != nil { + return nil, e + } else { + return &progress{ + fos: f, + b: new(atomic.Int32), + fi: new(atomic.Value), + fe: new(atomic.Value), + fr: new(atomic.Value), + }, nil + } +} + +func Open(name string) (Progress, error) { + if f, e := os.Open(name); e != nil { + return nil, e + } else { + return &progress{ + fos: f, + b: new(atomic.Int32), + fi: new(atomic.Value), + fe: new(atomic.Value), + fr: new(atomic.Value), + }, nil + } +} + +func Create(name string) (Progress, error) { + if f, e := os.Create(name); e != nil { + return nil, e + } else { + return &progress{ + fos: f, + b: new(atomic.Int32), + fi: new(atomic.Value), + fe: new(atomic.Value), + fr: new(atomic.Value), + }, nil + } +} diff --git a/file/progress/iobyte.go b/file/progress/iobyte.go new file mode 100644 index 0000000..fd0758d --- /dev/null +++ b/file/progress/iobyte.go @@ -0,0 +1,74 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package progress + +import ( + "errors" + "io" +) + +func (o *progress) ReadByte() (byte, error) { + var ( + p = make([]byte, 0, 1) + i int64 + n int + e error + ) + + if i, e = o.fos.Seek(0, io.SeekCurrent); e != nil && !errors.Is(e, io.EOF) { + return 0, e + } else if n, e = o.fos.Read(p); e != nil && !errors.Is(e, io.EOF) { + return 0, e + } else if n > 1 { + if _, e = o.fos.Seek(i+1, io.SeekStart); e != nil && !errors.Is(e, io.EOF) { + return 0, e + } + } + + return p[0], nil +} + +func (o *progress) WriteByte(c byte) error { + var ( + p = append(make([]byte, 0, 1), c) + i int64 + n int + e error + ) + + if i, e = o.fos.Seek(0, io.SeekCurrent); e != nil && !errors.Is(e, io.EOF) { + return e + } else if n, e = o.fos.Write(p); e != nil && !errors.Is(e, io.EOF) { + return e + } else if n > 1 { + if _, e = o.fos.Seek(i+1, io.SeekStart); e != nil && !errors.Is(e, io.EOF) { + return e + } + } + + return nil +} diff --git a/file/progress/iocloser.go b/file/progress/iocloser.go new file mode 100644 index 0000000..5f00a4b --- /dev/null +++ b/file/progress/iocloser.go @@ -0,0 +1,64 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package progress + +import "os" + +func (o *progress) clean(e error) error { + if o == nil { + return nil + } + + o.fos = nil + return e +} + +func (o *progress) Close() error { + if o == nil || o.fos == nil { + return nil + } + + return o.clean(o.fos.Close()) +} + +func (o *progress) CloseDelete() error { + if o == nil || o.fos == nil { + return nil + } + + n := o.Path() + + if e := o.clean(o.fos.Close()); e != nil { + return e + } + + if len(n) < 1 { + return nil + } + + return os.Remove(n) +} diff --git a/file/progress/ioreader.go b/file/progress/ioreader.go new file mode 100644 index 0000000..56317c5 --- /dev/null +++ b/file/progress/ioreader.go @@ -0,0 +1,108 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package progress + +import ( + "errors" + "io" + "math" +) + +func (o *progress) Read(p []byte) (n int, err error) { + if o == nil || o.fos == nil { + return 0, ErrorNilPointer.Error(nil) + } + + return o.analyze(o.fos.Read(p)) +} + +func (o *progress) ReadAt(p []byte, off int64) (n int, err error) { + if o == nil || o.fos == nil { + return 0, ErrorNilPointer.Error(nil) + } + + return o.analyze(o.fos.ReadAt(p, off)) +} + +func (o *progress) ReadFrom(r io.Reader) (n int64, err error) { + if o == nil || r == nil || o.fos == nil { + return 0, ErrorNilPointer.Error(nil) + } + + var size = o.getBufferSize(0) + + if l, ok := r.(*io.LimitedReader); ok && int64(size) > l.N { + if l.N < 1 { + size = 1 + } else if l.N > math.MaxInt { + size = math.MaxInt + } else { + size = int(l.N) + } + } + + var bf = make([]byte, o.getBufferSize(size)) + + for { + var ( + nr int + nw int + er error + ew error + ) + + // code from io.copy + + nr, er = r.Read(bf) + if nr > 0 { + nw, ew = o.Write(bf[:nr]) + } + + n += int64(nw) + + if er != nil && errors.Is(er, io.EOF) { + o.finish() + break + } else if er != nil { + err = er + break + } else if ew != nil { + err = ew + break + } else if nw < 0 || nw < nr { + err = errors.New("invalid write result") + break + } else if nr != nw { + err = io.ErrShortWrite + break + } + + clear(bf) + } + + return n, err +} diff --git a/file/progress/ioseeker.go b/file/progress/ioseeker.go new file mode 100644 index 0000000..8537008 --- /dev/null +++ b/file/progress/ioseeker.go @@ -0,0 +1,38 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package progress + +func (o *progress) Seek(offset int64, whence int) (int64, error) { + if o == nil || o.fos == nil { + return 0, ErrorNilPointer.Error(nil) + } + + n, err := o.fos.Seek(offset, whence) + o.reset() + + return n, err +} diff --git a/file/progress/iowriter.go b/file/progress/iowriter.go new file mode 100644 index 0000000..9e6e173 --- /dev/null +++ b/file/progress/iowriter.go @@ -0,0 +1,104 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package progress + +import ( + "errors" + "io" +) + +func (o *progress) Write(p []byte) (n int, err error) { + if o == nil || o.fos == nil { + return 0, ErrorNilPointer.Error(nil) + } + + return o.analyze(o.fos.Write(p)) +} + +func (o *progress) WriteAt(p []byte, off int64) (n int, err error) { + if o == nil || o.fos == nil { + return 0, ErrorNilPointer.Error(nil) + } + + return o.analyze(o.fos.WriteAt(p, off)) +} + +func (o *progress) WriteTo(w io.Writer) (n int64, err error) { + if o == nil || w == nil || o.fos == nil { + return 0, ErrorNilPointer.Error(nil) + } + + var bf = make([]byte, o.getBufferSize(0)) + + for { + var ( + nr int + nw int + er error + ew error + ) + + // code from io.copy + + nr, er = o.fos.Read(bf) + + if nr > 0 { + nw, ew = w.Write(bf[:nr]) + } + + n += int64(nw) + + if er != nil && errors.Is(er, io.EOF) { + o.finish() + break + } else if er != nil { + err = er + break + } else if ew != nil { + err = ew + break + } else if nw < nr { + err = io.ErrShortWrite + break + } else if nw != nr { + err = errors.New("invalid write result") + break + } + + clear(bf) + } + + return n, err +} + +func (o *progress) WriteString(s string) (n int, err error) { + if o == nil || o.fos == nil { + return 0, ErrorNilPointer.Error(nil) + } + + return o.analyze(o.fos.WriteString(s)) +} diff --git a/file/progress/model.go b/file/progress/model.go new file mode 100644 index 0000000..ac2aa41 --- /dev/null +++ b/file/progress/model.go @@ -0,0 +1,128 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package progress + +import ( + "io" + "os" + "path/filepath" + "sync/atomic" +) + +type progress struct { + fos *os.File + + b *atomic.Int32 + + fi *atomic.Value + fe *atomic.Value + fr *atomic.Value +} + +func (o *progress) SetBufferSize(size int32) { + o.b.Store(size) +} + +func (o *progress) getBufferSize(size int) int { + if size > 0 { + return size + } else if o == nil { + return DefaultBuffSize + } + + i := o.b.Load() + if i < 1024 { + return DefaultBuffSize + } else { + return int(i) + } +} + +func (o *progress) Path() string { + return filepath.Clean(o.fos.Name()) +} + +func (o *progress) Stat() (os.FileInfo, error) { + if o == nil || o.fos == nil { + return nil, ErrorNilPointer.Error(nil) + } + + if i, e := o.fos.Stat(); e != nil { + return i, ErrorIOFileStat.ErrorParent(e) + } else { + return i, nil + } +} + +func (o *progress) SizeBOF() (size int64, err error) { + if o == nil || o.fos == nil { + return 0, ErrorNilPointer.Error(nil) + } + + return o.Seek(0, io.SeekCurrent) +} + +func (o *progress) SizeEOF() (size int64, err error) { + if o == nil || o.fos == nil { + return 0, ErrorNilPointer.Error(nil) + } + + var ( + e error + a int64 // origin + b int64 // eof + ) + + if a, e = o.Seek(0, io.SeekCurrent); e != nil { + return 0, e + } else if b, e = o.Seek(0, io.SeekEnd); e != nil { + return 0, e + } else if _, e = o.Seek(a, io.SeekStart); e != nil { + return 0, e + } else { + return b - a, nil + } +} + +func (o *progress) Truncate(size int64) error { + if o == nil || o.fos == nil { + return ErrorNilPointer.Error(nil) + } + + e := o.fos.Truncate(size) + o.reset() + + return e +} + +func (o *progress) Sync() error { + if o == nil || o.fos == nil { + return ErrorNilPointer.Error(nil) + } + + return o.fos.Sync() +} diff --git a/file/progress/progress.go b/file/progress/progress.go new file mode 100644 index 0000000..d95fc9f --- /dev/null +++ b/file/progress/progress.go @@ -0,0 +1,176 @@ +/* + * MIT License + * + * Copyright (c) 2020 Nicolas JUHEL + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + */ + +package progress + +import ( + "errors" + "io" +) + +func (o *progress) RegisterFctIncrement(fct FctIncrement) { + o.fi.Store(fct) +} + +func (o *progress) RegisterFctReset(fct FctReset) { + o.fr.Store(fct) +} + +func (o *progress) RegisterFctEOF(fct FctEOF) { + o.fe.Store(fct) +} + +func (o *progress) SetRegisterProgress(f Progress) { + i := o.fi.Load() + if i != nil { + f.RegisterFctIncrement(i.(FctIncrement)) + } + + i = o.fr.Load() + if i != nil { + f.RegisterFctReset(i.(FctReset)) + } + + i = o.fe.Load() + if i != nil { + f.RegisterFctEOF(i.(FctEOF)) + } +} + +func (o *progress) inc(n int64) { + if o == nil { + return + } + + f := o.fi.Load() + if f != nil { + f.(FctIncrement)(n) + } +} + +func (o *progress) incN(n int64, s int) { + if o == nil { + return + } + + f := o.fi.Load() + if f != nil { + f.(FctIncrement)(n + int64(s)) + } +} + +func (o *progress) dec(n int64) { + if o == nil { + return + } + + f := o.fi.Load() + if f != nil { + f.(FctIncrement)(0 - n) + } +} + +func (o *progress) decN(n int64, s int) { + if o == nil { + return + } + + f := o.fi.Load() + if f != nil { + f.(FctIncrement)(0 - (n + int64(s))) + } +} + +func (o *progress) finish() { + if o == nil { + return + } + + f := o.fe.Load() + if f != nil { + f.(FctEOF)() + } +} + +func (o *progress) reset() { + o.Reset(0) +} + +func (o *progress) Reset(max int64) { + if o == nil { + return + } + + f := o.fr.Load() + + if f != nil { + if max < 1 { + if i, e := o.Stat(); e != nil { + return + } else { + max = i.Size() + } + } + + if s, e := o.SizeBOF(); e != nil { + return + } else if s >= 0 { + f.(FctReset)(max, s) + } + } +} + +func (o *progress) analyze(i int, e error) (n int, err error) { + if o == nil { + return i, e + } + + if i != 0 { + o.inc(int64(i)) + } + + if e != nil && errors.Is(e, io.EOF) { + o.finish() + } + + return i, e +} + +func (o *progress) analyze64(i int64, e error) (n int64, err error) { + if o == nil { + return i, e + } + + if i != 0 { + o.inc(i) + } + + if e != nil && errors.Is(e, io.EOF) { + o.finish() + } + + return i, e +} diff --git a/go.mod b/go.mod index d005d3b..33dc2ec 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,23 @@ module github.com/nabbar/golib -go 1.21.0 +go 1.21 + +toolchain go1.21.0 require ( - github.com/aws/aws-sdk-go-v2 v1.20.1 - github.com/aws/aws-sdk-go-v2/config v1.18.33 - github.com/aws/aws-sdk-go-v2/credentials v1.13.32 - github.com/aws/aws-sdk-go-v2/service/iam v1.22.2 - github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 + github.com/aws/aws-sdk-go-v2 v1.21.0 + github.com/aws/aws-sdk-go-v2/config v1.18.36 + github.com/aws/aws-sdk-go-v2/credentials v1.13.35 + github.com/aws/aws-sdk-go-v2/service/iam v1.22.5 + github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 github.com/bits-and-blooms/bitset v1.8.0 github.com/c-bata/go-prompt v0.2.6 github.com/fatih/color v1.15.0 github.com/fsnotify/fsnotify v1.6.0 - github.com/fxamacker/cbor/v2 v2.4.0 + github.com/fxamacker/cbor/v2 v2.5.0 github.com/gin-gonic/gin v1.9.1 github.com/go-ldap/ldap/v3 v3.4.5 - github.com/go-playground/validator/v10 v10.15.0 + github.com/go-playground/validator/v10 v10.15.1 github.com/google/go-github/v33 v33.0.0 github.com/hashicorp/go-hclog v1.5.0 github.com/hashicorp/go-retryablehttp v0.7.4 @@ -27,7 +29,7 @@ require ( github.com/mattn/go-colorable v0.1.13 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/nats-io/jwt/v2 v2.4.1 + github.com/nats-io/jwt/v2 v2.5.0 github.com/nats-io/nats-server/v2 v2.9.21 github.com/nats-io/nats.go v1.28.0 github.com/nutsdb/nutsdb v0.13.1 @@ -45,7 +47,7 @@ require ( github.com/xanzy/go-gitlab v0.90.0 github.com/xhit/go-simple-mail v2.2.2+incompatible github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 - golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb + golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/net v0.14.0 golang.org/x/oauth2 v0.11.0 golang.org/x/sync v0.3.0 @@ -55,15 +57,15 @@ require ( gorm.io/driver/clickhouse v0.5.1 gorm.io/driver/mysql v1.5.1 gorm.io/driver/postgres v1.5.2 - gorm.io/driver/sqlite v1.5.2 + gorm.io/driver/sqlite v1.5.3 gorm.io/driver/sqlserver v1.5.1 - gorm.io/gorm v1.25.3 + gorm.io/gorm v1.25.4 ) require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect - github.com/ClickHouse/ch-go v0.58.0 // indirect - github.com/ClickHouse/clickhouse-go/v2 v2.13.0 // indirect + github.com/ClickHouse/ch-go v0.58.2 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.13.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible // indirect @@ -74,20 +76,20 @@ require ( github.com/andybalholm/brotli v1.0.5 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/armon/go-metrics v0.4.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 // indirect - github.com/aws/smithy-go v1.14.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 // indirect github.com/bytedance/sonic v1.10.0 // indirect @@ -121,8 +123,8 @@ require ( github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/pprof v0.0.0-20230811205829-9131a7e9cc17 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect + github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -184,7 +186,7 @@ require ( github.com/spf13/cast v1.5.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect - github.com/subosito/gotenv v1.4.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/valyala/fastrand v1.1.0 // indirect @@ -201,7 +203,7 @@ require ( golang.org/x/mod v0.12.0 // indirect golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/ioutils/fileProgess.go b/ioutils/fileProgess.go deleted file mode 100644 index f64c716..0000000 --- a/ioutils/fileProgess.go +++ /dev/null @@ -1,579 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Nicolas JUHEL - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -package ioutils - -import ( - "bufio" - "context" - "io" - "os" - "path/filepath" - - liberr "github.com/nabbar/golib/errors" -) - -const buffSize = 32 * 1024 // see io.copyBuffer - -type FileProgress interface { - io.ReadCloser - io.ReadSeeker - io.ReadWriteCloser - io.ReadWriteSeeker - io.WriteCloser - io.WriteSeeker - io.Reader - io.ReaderFrom - io.ReaderAt - io.Writer - io.WriterAt - io.WriterTo - io.Seeker - io.StringWriter - io.Closer - - GetByteReader() io.ByteReader - GetByteWriter() io.ByteWriter - - SetContext(ctx context.Context) - SetIncrement(increment func(size int64)) - SetFinish(finish func()) - SetReset(reset func(size, current int64)) - ResetMax(max int64) - - FilePath() string - FileStat() (os.FileInfo, liberr.Error) - - SizeToEOF() (size int64, err liberr.Error) - Truncate(size int64) liberr.Error - Sync() liberr.Error - - NewFilePathMode(filepath string, mode int, perm os.FileMode) (FileProgress, liberr.Error) - NewFilePathWrite(filepath string, create, overwrite bool, perm os.FileMode) (FileProgress, liberr.Error) - NewFilePathRead(filepath string, perm os.FileMode) (FileProgress, liberr.Error) - NewFileTemp() (FileProgress, liberr.Error) - - CloseNoClean() error -} - -func NewFileProgress(file *os.File) FileProgress { - return &fileProgress{ - fs: file, - fc: nil, - ff: nil, - fr: nil, - t: false, - x: nil, - } -} - -func NewFileProgressTemp() (FileProgress, liberr.Error) { - if fs, e := NewTempFile(); e != nil { - return nil, e - } else { - return &fileProgress{ - fs: fs, - fc: nil, - ff: nil, - fr: nil, - t: true, - x: nil, - }, nil - } -} - -func NewFileProgressPathOpen(filepath string) (FileProgress, liberr.Error) { - //nolint #gosec - /* #nosec */ - if f, err := os.Open(filepath); err != nil { - return nil, ErrorIOFileOpen.ErrorParent(err) - } else { - return NewFileProgress(f), nil - } -} - -func NewFileProgressPathMode(filepath string, mode int, perm os.FileMode) (FileProgress, liberr.Error) { - //nolint #nosec - /* #nosec */ - if f, err := os.OpenFile(filepath, mode, perm); err != nil { - return nil, ErrorIOFileOpen.ErrorParent(err) - } else { - return NewFileProgress(f), nil - } -} - -func NewFileProgressPathWrite(filepath string, create, overwrite bool, perm os.FileMode) (FileProgress, liberr.Error) { - var mode = os.O_RDWR - - if _, err := os.Stat(filepath); err != nil && os.IsNotExist(err) && create { - mode = os.O_RDWR | os.O_CREATE - } else if err != nil { - return nil, ErrorIOFileStat.ErrorParent(err) - } else if err == nil && overwrite { - mode = os.O_RDWR | os.O_TRUNC - } - - return NewFileProgressPathMode(filepath, mode, perm) -} - -func NewFileProgressPathRead(filepath string, perm os.FileMode) (FileProgress, liberr.Error) { - mode := os.O_RDONLY - - if _, err := os.Stat(filepath); err != nil && !os.IsExist(err) { - return nil, ErrorIOFileStat.ErrorParent(err) - } - - return NewFileProgressPathMode(filepath, mode, perm) -} - -type fileProgress struct { - t bool - x context.Context - fs *os.File - fc func(size int64) - ff func() - fr func(size, current int64) -} - -func (f *fileProgress) NewFilePathMode(filepath string, mode int, perm os.FileMode) (FileProgress, liberr.Error) { - if f == nil { - return nil, ErrorNilPointer.Error(nil) - } - - if fs, e := NewFileProgressPathMode(filepath, mode, perm); e != nil { - return nil, e - } else { - fs.SetContext(f.x) - fs.SetFinish(f.ff) - fs.SetIncrement(f.fc) - fs.SetReset(f.fr) - return fs, nil - } -} - -func (f *fileProgress) NewFilePathWrite(filepath string, create, overwrite bool, perm os.FileMode) (FileProgress, liberr.Error) { - if f == nil { - return nil, ErrorNilPointer.Error(nil) - } - - if fs, e := NewFileProgressPathWrite(filepath, create, overwrite, perm); e != nil { - return nil, e - } else { - fs.SetContext(f.x) - fs.SetFinish(f.ff) - fs.SetIncrement(f.fc) - fs.SetReset(f.fr) - return fs, nil - } -} - -func (f *fileProgress) NewFilePathRead(filepath string, perm os.FileMode) (FileProgress, liberr.Error) { - if f == nil { - return nil, ErrorNilPointer.Error(nil) - } - - if fs, e := NewFileProgressPathRead(filepath, perm); e != nil { - return nil, e - } else { - fs.SetContext(f.x) - fs.SetFinish(f.ff) - fs.SetIncrement(f.fc) - fs.SetReset(f.fr) - return fs, nil - } -} - -func (f *fileProgress) NewFileTemp() (FileProgress, liberr.Error) { - if f == nil { - return nil, ErrorNilPointer.Error(nil) - } - - if fs, e := NewFileProgressTemp(); e != nil { - return nil, e - } else { - fs.SetContext(f.x) - fs.SetFinish(f.ff) - fs.SetIncrement(f.fc) - fs.SetReset(f.fr) - return fs, nil - } -} - -func (f *fileProgress) SetContext(ctx context.Context) { - if f != nil { - f.x = ctx - } -} - -func (f *fileProgress) checkContext() error { - if f == nil { - return ErrorNilPointer.Error(nil) - } - - if f.x != nil && f.x.Err() != nil { - return f.x.Err() - } - - return nil -} - -func (f *fileProgress) FilePath() string { - if f == nil || f.fs == nil { - return "" - } - - return filepath.Clean(f.fs.Name()) -} - -func (f *fileProgress) FileStat() (os.FileInfo, liberr.Error) { - if f == nil { - return nil, ErrorNilPointer.Error(nil) - } - - if i, e := f.fs.Stat(); e != nil { - return i, ErrorIOFileStat.ErrorParent(e) - } else { - return i, nil - } -} - -func (f *fileProgress) SizeToEOF() (size int64, err liberr.Error) { - if f == nil { - return 0, ErrorNilPointer.Error(nil) - } - - if a, e := f.Seek(0, io.SeekCurrent); e != nil { - return 0, ErrorIOFileSeek.ErrorParent(e) - } else if b, e := f.Seek(0, io.SeekEnd); e != nil { - return 0, ErrorIOFileSeek.ErrorParent(e) - } else if _, e := f.Seek(a, io.SeekStart); e != nil { - return 0, ErrorIOFileSeek.ErrorParent(e) - } else { - return b - a, nil - } -} - -func (f *fileProgress) Truncate(size int64) liberr.Error { - if f == nil { - return ErrorNilPointer.Error(nil) - } - - if e := f.fs.Truncate(size); e != nil { - return ErrorIOFileTruncate.ErrorParent(e) - } - - f.reset(0) - - return nil -} - -func (f *fileProgress) Sync() liberr.Error { - if f == nil { - return ErrorNilPointer.Error(nil) - } - - if e := f.fs.Sync(); e != nil { - return ErrorIOFileSync.ErrorParent(e) - } - - return nil -} - -func (f *fileProgress) SetIncrement(increment func(size int64)) { - if f != nil { - f.fc = increment - } -} - -func (f *fileProgress) SetFinish(finish func()) { - if f != nil { - f.ff = finish - } -} - -func (f *fileProgress) SetReset(reset func(size, current int64)) { - if f != nil { - f.fr = reset - } -} - -func (f *fileProgress) GetByteReader() io.ByteReader { - if f == nil { - return nil - } - - return bufio.NewReaderSize(f, buffSize) -} - -func (f *fileProgress) GetByteWriter() io.ByteWriter { - if f == nil { - return nil - } - - return bufio.NewWriterSize(f, buffSize) -} - -func (f *fileProgress) ReadFrom(r io.Reader) (n int64, err error) { - if f == nil || r == nil { - return 0, ErrorNilPointer.Error(nil) - } - - size := buffSize - if l, ok := r.(*io.LimitedReader); ok && int64(size) > l.N { - if l.N < 1 { - size = 1 - } else { - size = int(l.N) - } - } - buf := make([]byte, size) - - for { - if err = f.checkContext(); err != nil { - break - } - - // copy from bufio CopyBuff - nr, er := r.Read(buf) - - if nr > 0 { - nw, ew := f.Write(buf[0:nr]) - if nw > 0 { - n += int64(nw) - } - if ew != nil { - err = ew - break - } - if nr != nw { - err = io.ErrShortWrite - break - } - } - if er != nil { - if er != io.EOF { - err = er - } - break - } - } - - f.finish() - return n, err -} - -func (f *fileProgress) WriteTo(w io.Writer) (n int64, err error) { - if f == nil { - return 0, ErrorNilPointer.Error(nil) - } - - buf := make([]byte, buffSize) - - for { - if err = f.checkContext(); err != nil { - break - } - - // copy from bufio CopyBuff - nr, er := f.Read(buf) - - if nr > 0 { - nw, ew := w.Write(buf[0:nr]) - if nw > 0 { - n += int64(nw) - } - if ew != nil { - err = ew - break - } - if nr != nw { - err = io.ErrShortWrite - break - } - } - if er != nil { - if er != io.EOF { - err = er - } - break - } - } - - f.finish() - return n, err -} - -// nolint #unparam -func (f *fileProgress) increment(n int64, size int) { - n += int64(size) - - if f != nil && f.fc != nil && n > 0 { - f.fc(n) - } -} - -func (f *fileProgress) finish() { - if f != nil && f.ff != nil { - f.ff() - } -} - -func (f *fileProgress) reset(pos int64) { - if f.fr != nil { - if i, e := f.FileStat(); e != nil { - return - } else if i.Size() != 0 { - f.fr(i.Size(), pos) - } - } -} - -func (f *fileProgress) ResetMax(max int64) { - if f.fr != nil { - f.fr(max, 0) - } -} - -func (f *fileProgress) Read(p []byte) (n int, err error) { - if f == nil { - return 0, ErrorNilPointer.Error(nil) - } - - n, err = f.fs.Read(p) - //logger.DebugLevel.Logf("Loading from '%s' '%d' bytes, with err '%v'", f.FilePath(), n, err) - - if err != nil && err == io.EOF { - f.finish() - } else if err == nil { - f.increment(0, n) - } - - return -} - -func (f *fileProgress) ReadAt(p []byte, off int64) (n int, err error) { - if f == nil { - return 0, ErrorNilPointer.Error(nil) - } - - n, err = f.fs.ReadAt(p, off) - - if err != nil && err == io.EOF { - f.finish() - } else if err == nil { - f.increment(0, n) - } - - return -} - -func (f *fileProgress) Write(p []byte) (n int, err error) { - if f == nil { - return 0, ErrorNilPointer.Error(nil) - } - - n, err = f.fs.Write(p) - //logger.DebugLevel.Logf("Writing from '%s' '%d' bytes, with err '%v'", f.FilePath(), n, err) - - if err != nil && err == io.EOF { - f.finish() - } else if err == nil { - f.increment(0, n) - } - - return -} - -func (f *fileProgress) WriteAt(p []byte, off int64) (n int, err error) { - if f == nil { - return 0, ErrorNilPointer.Error(nil) - } - - n, err = f.fs.WriteAt(p, off) - - if err != nil && err == io.EOF { - f.finish() - } else if err == nil { - f.increment(0, n) - } - - return -} - -func (f *fileProgress) Seek(offset int64, whence int) (n int64, err error) { - if f == nil { - return 0, ErrorNilPointer.Error(nil) - } - - n, err = f.fs.Seek(offset, whence) - - if err != nil && err != io.EOF { - return - } - - f.reset(n) - - return -} - -func (f *fileProgress) WriteString(s string) (n int, err error) { - if f == nil { - return 0, ErrorNilPointer.Error(nil) - } - - n, err = f.fs.WriteString(s) - - if err != nil && err == io.EOF { - f.finish() - } else if err == nil { - f.increment(0, n) - } - - return -} - -func (f *fileProgress) clean(err error) error { - f.ff = nil - f.fr = nil - f.fc = nil - f.fs = nil - return err -} - -func (f *fileProgress) Close() error { - if f == nil || f.fs == nil { - return nil - } else if f.t { - return f.clean(DelTempFile(f.fs)) - } else if f.fs != nil { - return f.clean(f.fs.Close()) - } - - return nil -} - -func (f *fileProgress) CloseNoClean() error { - return f.clean(f.fs.Close()) -} diff --git a/logger/logger_suite_test.go b/logger/logger_suite_test.go index 914aa30..d3705d3 100644 --- a/logger/logger_suite_test.go +++ b/logger/logger_suite_test.go @@ -32,7 +32,7 @@ import ( "os" "testing" - "github.com/nabbar/golib/ioutils" + libfpg "github.com/nabbar/golib/file/progress" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -66,16 +66,18 @@ func GetContext() context.Context { } func GetTempFile() (string, error) { - hdf, err := ioutils.NewFileProgressTemp() + hdf, err := libfpg.Temp("") if err != nil { return "", err } defer func() { - _ = hdf.Close() + if hdf != nil { + _ = hdf.CloseDelete() + } }() - return hdf.FilePath(), nil + return hdf.Path(), nil } func DelTempFile(filepath string) error { diff --git a/mail/config.go b/mail/config.go index 379ca06..09f85fb 100644 --- a/mail/config.go +++ b/mail/config.go @@ -32,7 +32,7 @@ import ( libval "github.com/go-playground/validator/v10" liberr "github.com/nabbar/golib/errors" - libiot "github.com/nabbar/golib/ioutils" + libfpg "github.com/nabbar/golib/file/progress" ) type Config struct { @@ -169,8 +169,8 @@ func (c Config) NewMailer() (Mail, liberr.Error) { if len(c.Attach) > 0 { for _, f := range c.Attach { - if h, e := libiot.NewFileProgressPathRead(f.Path, 0); e != nil { - return nil, e + if h, e := libfpg.Open(f.Path); e != nil { + return nil, ErrorFileOpenCreate.ErrorParent(e) } else { m.AddAttachment(f.Name, f.Mime, h, false) } @@ -179,8 +179,8 @@ func (c Config) NewMailer() (Mail, liberr.Error) { if len(c.Inline) > 0 { for _, f := range c.Inline { - if h, e := libiot.NewFileProgressPathRead(f.Path, 0); e != nil { - return nil, e + if h, e := libfpg.Open(f.Path); e != nil { + return nil, ErrorFileOpenCreate.ErrorParent(e) } else { m.AddAttachment(f.Name, f.Mime, h, true) } diff --git a/mail/error.go b/mail/error.go index 4b7bbc1..47099c6 100644 --- a/mail/error.go +++ b/mail/error.go @@ -32,6 +32,8 @@ import ( liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/mail" + const ( ErrorParamEmpty liberr.CodeError = iota + liberr.MinPkgMail ErrorMailConfigInvalid @@ -40,11 +42,12 @@ const ( ErrorMailDateParsing ErrorMailSmtpClient ErrorMailSenderInit + ErrorFileOpenCreate ) func init() { if liberr.ExistInMapMessage(ErrorParamEmpty) { - panic(fmt.Errorf("error code collision with package golib/mail")) + panic(fmt.Errorf("error code collision with package %s", pkgName)) } liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } @@ -65,6 +68,8 @@ func getMessage(code liberr.CodeError) (message string) { return "error occurs while to checking connection with SMTP server" case ErrorMailSenderInit: return "error occurs while to preparing SMTP Email sender" + case ErrorFileOpenCreate: + return "cannot open/create file" } return liberr.NullMessage diff --git a/mail/sender.go b/mail/sender.go index 86da666..d829080 100644 --- a/mail/sender.go +++ b/mail/sender.go @@ -32,7 +32,7 @@ import ( "io" liberr "github.com/nabbar/golib/errors" - libiot "github.com/nabbar/golib/ioutils" + libfpg "github.com/nabbar/golib/file/progress" libsmtp "github.com/nabbar/golib/smtp" simple "github.com/xhit/go-simple-mail" ) @@ -48,7 +48,7 @@ type Sender interface { } type sender struct { - data libiot.FileProgress + data libfpg.Progress from string rcpt []string } @@ -56,7 +56,7 @@ type sender struct { // nolint #gocognit func (m *mail) Sender() (snd Sender, err liberr.Error) { e := simple.NewMSG() - f := make([]libiot.FileProgress, 0) + f := make([]libfpg.Progress, 0) switch m.GetPriority() { case PriorityHigh: @@ -145,11 +145,11 @@ func (m *mail) Sender() (snd Sender, err liberr.Error) { if len(m.attach) > 0 { for _, i := range m.attach { - if t, er := libiot.NewFileProgressTemp(); er != nil { - return nil, er + if t, er := libfpg.Temp(""); er != nil { + return nil, ErrorFileOpenCreate.ErrorParent(er) } else if _, er := t.ReadFrom(i.data); er != nil { return nil, ErrorMailIORead.ErrorParent(er) - } else if e.AddAttachment(t.FilePath(), i.name); e.Error != nil { + } else if e.AddAttachment(t.Path(), i.name); e.Error != nil { return nil, ErrorMailSenderInit.ErrorParent(e.Error) } else { f = append(f, t) @@ -159,11 +159,11 @@ func (m *mail) Sender() (snd Sender, err liberr.Error) { if len(m.inline) > 0 { for _, i := range m.inline { - if t, er := libiot.NewFileProgressTemp(); er != nil { - return nil, er + if t, er := libfpg.Temp(""); er != nil { + return nil, ErrorFileOpenCreate.ErrorParent(er) } else if _, er := t.ReadFrom(i.data); er != nil { return nil, ErrorMailIORead.ErrorParent(er) - } else if e.AddInline(t.FilePath(), i.name); e.Error != nil { + } else if e.AddInline(t.Path(), i.name); e.Error != nil { return nil, ErrorMailSenderInit.ErrorParent(e.Error) } else { f = append(f, t) @@ -209,9 +209,9 @@ func (m *mail) Sender() (snd Sender, err liberr.Error) { s.rcpt = append(s.rcpt, m.Email().GetRecipients(RecipientCC)...) s.rcpt = append(s.rcpt, m.Email().GetRecipients(RecipientBCC)...) - if tmp, err := libiot.NewFileProgressTemp(); err != nil { - return nil, err - } else if _, er := tmp.WriteString(e.GetMessage()); er != nil { + if tmp, er := libfpg.Temp(""); er != nil { + return nil, ErrorFileOpenCreate.ErrorParent(er) + } else if _, er = tmp.WriteString(e.GetMessage()); er != nil { return nil, ErrorMailIOWrite.ErrorParent(er) } else if e.Error != nil { return nil, ErrorMailSenderInit.ErrorParent(e.Error) diff --git a/nutsdb/client.go b/nutsdb/client.go index dbb67e8..676b34c 100644 --- a/nutsdb/client.go +++ b/nutsdb/client.go @@ -96,93 +96,93 @@ func (c *clientNutDB) strToType(dest reflect.Type, val string) (interface{}, lib switch dest.Kind() { case reflect.Bool: if v, e := strconv.ParseBool(val); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return v, nil } case reflect.Int: if v, e := strconv.ParseInt(val, 10, 64); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return int(v), nil } case reflect.Int8: if v, e := strconv.ParseInt(val, 10, 8); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return int8(v), nil } case reflect.Int16: if v, e := strconv.ParseInt(val, 10, 16); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return int16(v), nil } case reflect.Int32: if v, e := strconv.ParseInt(val, 10, 32); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return int32(v), nil } case reflect.Int64: if v, e := strconv.ParseInt(val, 10, 64); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return v, nil } case reflect.Uint: if v, e := strconv.ParseUint(val, 10, 64); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return uint(v), nil } case reflect.Uint8: if v, e := strconv.ParseUint(val, 10, 8); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return uint8(v), nil } case reflect.Uint16: if v, e := strconv.ParseUint(val, 10, 16); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return uint16(v), nil } case reflect.Uint32: if v, e := strconv.ParseUint(val, 10, 32); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return uint32(v), nil } case reflect.Uint64: if v, e := strconv.ParseUint(val, 10, 64); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return v, nil } case reflect.Uintptr: - return nil, ErrorParamsInvalid.ErrorParent(fmt.Errorf("cannot convert int UintPtr")) + return nil, ErrorParamInvalid.ErrorParent(fmt.Errorf("cannot convert int UintPtr")) case reflect.Float32: if v, e := strconv.ParseFloat(val, 32); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return float32(v), nil } case reflect.Float64: if v, e := strconv.ParseFloat(val, 64); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return v, nil } case reflect.Complex64: if v, e := strconv.ParseComplex(val, 64); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return complex64(v), nil } case reflect.Complex128: if v, e := strconv.ParseComplex(val, 128); e != nil { - return nil, ErrorParamsMismatching.ErrorParent(e) + return nil, ErrorParamMismatching.ErrorParent(e) } else { return v, nil } @@ -190,12 +190,12 @@ func (c *clientNutDB) strToType(dest reflect.Type, val string) (interface{}, lib if dest == sliceByte.Type() { return []byte(val), nil } else { - return nil, ErrorParamsInvalid.Error(nil) + return nil, ErrorParamInvalid.Error(nil) } case reflect.String: return val, nil default: - return nil, ErrorParamsInvalid.Error(nil) + return nil, ErrorParamInvalid.Error(nil) } } @@ -206,15 +206,15 @@ func (c *clientNutDB) Run(cmd CmdCode, args []string) (*CommandResponse, liberr. switch cmd { case CmdZCount: if len(args) < 3 || len(args) > 6 { - return nil, ErrorParamsInvalidNumber.Error(nil) + return nil, ErrorParamInvalidNumber.Error(nil) } case CmdZRangeByScore: if len(args) < 3 || len(args) > 6 { - return nil, ErrorParamsInvalidNumber.Error(nil) + return nil, ErrorParamInvalidNumber.Error(nil) } default: if len(args) != nbPrm { - return nil, ErrorParamsInvalidNumber.Error(nil) + return nil, ErrorParamInvalidNumber.Error(nil) } } @@ -235,7 +235,7 @@ func (c *clientNutDB) Run(cmd CmdCode, args []string) (*CommandResponse, liberr. } else if v == nil { opt.Limit = 0 } else if vv, ok := v.(int); !ok { - return nil, ErrorParamsMismatching.Error(nil) + return nil, ErrorParamMismatching.Error(nil) } else { opt.Limit = vv } @@ -245,7 +245,7 @@ func (c *clientNutDB) Run(cmd CmdCode, args []string) (*CommandResponse, liberr. } else if v == nil { opt.ExcludeStart = false } else if vv, ok := v.(bool); !ok { - return nil, ErrorParamsMismatching.Error(nil) + return nil, ErrorParamMismatching.Error(nil) } else { opt.ExcludeStart = vv } @@ -255,12 +255,12 @@ func (c *clientNutDB) Run(cmd CmdCode, args []string) (*CommandResponse, liberr. } else if v == nil { opt.ExcludeEnd = false } else if vv, ok := v.(bool); !ok { - return nil, ErrorParamsMismatching.Error(nil) + return nil, ErrorParamMismatching.Error(nil) } else { opt.ExcludeEnd = vv } default: - return nil, ErrorParamsInvalid.Error(nil) + return nil, ErrorParamInvalid.Error(nil) } default: if v, e := c.strToType(method.Type().In(i), args[i]); e != nil { diff --git a/nutsdb/errors.go b/nutsdb/errors.go index 1775a97..e66659d 100644 --- a/nutsdb/errors.go +++ b/nutsdb/errors.go @@ -31,18 +31,23 @@ package nutsdb import ( + "fmt" + liberr "github.com/nabbar/golib/errors" ) +const pkgName = "golib/nutsdb" + const ( - ErrorParamsEmpty liberr.CodeError = iota + liberr.MinPkgNutsDB - ErrorParamsMissing - ErrorParamsMismatching - ErrorParamsInvalid - ErrorParamsInvalidNumber + ErrorParamEmpty liberr.CodeError = iota + liberr.MinPkgNutsDB + ErrorParamMissing + ErrorParamMismatching + ErrorParamInvalid + ErrorParamInvalidNumber ErrorValidateConfig ErrorValidateNutsDB ErrorClusterInit + ErrorFileTemp ErrorFolderCheck ErrorFolderCreate ErrorFolderCopy @@ -72,30 +77,26 @@ const ( ErrorClientCommandResponseInvalid ) -var isCodeError = false - -func IsCodeError() bool { - return isCodeError -} - func init() { - isCodeError = liberr.ExistInMapMessage(ErrorParamsEmpty) - liberr.RegisterIdFctMessage(ErrorParamsEmpty, getMessage) + if liberr.ExistInMapMessage(ErrorParamEmpty) { + panic(fmt.Errorf("error code collision %s", pkgName)) + } + liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage) } func getMessage(code liberr.CodeError) (message string) { switch code { - case liberr.UNK_ERROR: - return "" - case ErrorParamsEmpty: + case liberr.UnknownError: + return liberr.NullMessage + case ErrorParamEmpty: return "at least one given parameter is empty" - case ErrorParamsMissing: + case ErrorParamMissing: return "at least one given parameter is missing" - case ErrorParamsMismatching: + case ErrorParamMismatching: return "at least one given parameter does not match the awaiting type" - case ErrorParamsInvalid: + case ErrorParamInvalid: return "at least one given parameter is invalid" - case ErrorParamsInvalidNumber: + case ErrorParamInvalidNumber: return "the number of parameters is not matching the awaiting number" case ErrorValidateConfig: return "config seems to be invalid" @@ -103,6 +104,8 @@ func getMessage(code liberr.CodeError) (message string) { return "database config seems to be invalid" case ErrorClusterInit: return "cannot start or join cluster" + case ErrorFileTemp: + return "error while trying to create new temp file" case ErrorFolderCheck: return "error while trying to check or stat folder" case ErrorFolderCreate: @@ -157,5 +160,5 @@ func getMessage(code liberr.CodeError) (message string) { return "response of requested client command seems to be invalid" } - return "" + return liberr.NullMessage } diff --git a/nutsdb/node.go b/nutsdb/node.go index 9e0e2ce..ad580f7 100644 --- a/nutsdb/node.go +++ b/nutsdb/node.go @@ -337,9 +337,9 @@ func (n *nutsNode) PrepareSnapshot() (interface{}, error) { func (n *nutsNode) SaveSnapshot(i interface{}, writer io.Writer, c <-chan struct{}) error { if i == nil { - return ErrorParamsEmpty.Error(nil) + return ErrorParamEmpty.Error(nil) } else if sh, ok := snapCast(i); !ok { - return ErrorParamsMismatching.Error(nil) + return ErrorParamMismatching.Error(nil) } else if opt := n.getOptions(); opt == nil { return ErrorValidateConfig.Error(nil) } else if err := sh.Save(opt, writer); err != nil { diff --git a/nutsdb/options.go b/nutsdb/options.go index fc741df..7741de4 100644 --- a/nutsdb/options.go +++ b/nutsdb/options.go @@ -33,7 +33,6 @@ package nutsdb import ( "bytes" "fmt" - "io/ioutil" "os" "path/filepath" "runtime" @@ -42,8 +41,8 @@ import ( "time" liberr "github.com/nabbar/golib/errors" - "github.com/nutsdb/nutsdb" - "github.com/xujiajun/utils/filesystem" + nutsdb "github.com/nutsdb/nutsdb" + xujufs "github.com/xujiajun/utils/filesystem" ) type Options interface { @@ -53,7 +52,8 @@ type Options interface { NewBackupTemp(db *nutsdb.DB) (string, liberr.Error) NewTempFolder() (string, liberr.Error) - NewTempFile(extension string) (string, liberr.Error) + NewTempFilePattern(extension string) string + GetTempDir() string CleanBackup() liberr.Error Permission() os.FileMode @@ -132,7 +132,7 @@ func (o options) NewBackupTemp(db *nutsdb.DB) (string, liberr.Error) { } func (o options) NewTempFolder() (string, liberr.Error) { - if p, e := ioutil.TempDir(o.dirs.temp, o.getTempPrefix()); e != nil { + if p, e := os.MkdirTemp(o.dirs.temp, o.getTempPrefix()); e != nil { return "", ErrorFolderCreate.ErrorParent(e) } else { _ = os.Chmod(p, o.perm) @@ -140,21 +140,18 @@ func (o options) NewTempFolder() (string, liberr.Error) { } } -func (o options) NewTempFile(extension string) (string, liberr.Error) { +func (o options) NewTempFilePattern(extension string) string { pattern := o.getTempPrefix() + "-*" if extension != "" { pattern = pattern + "." + extension } - if file, e := ioutil.TempFile(o.dirs.temp, pattern); e != nil { - return "", ErrorFolderCreate.ErrorParent(e) - } else { - p := file.Name() - _ = file.Close() + return pattern +} - return p, nil - } +func (o options) GetTempDir() string { + return o.dirs.temp } func (o options) CleanBackup() liberr.Error { @@ -168,7 +165,7 @@ func (o options) Permission() os.FileMode { func (o options) RestoreBackup(dir string) liberr.Error { if err := os.RemoveAll(o.dirs.data); err != nil { return ErrorFolderDelete.ErrorParent(err) - } else if err = filesystem.CopyDir(dir, o.dirs.data); err != nil { + } else if err = xujufs.CopyDir(dir, o.dirs.data); err != nil { return ErrorFolderCopy.ErrorParent(err) } else { _ = os.Chmod(o.dirs.data, o.perm) diff --git a/nutsdb/snap.go b/nutsdb/snap.go index 831cf51..b96df5a 100644 --- a/nutsdb/snap.go +++ b/nutsdb/snap.go @@ -35,9 +35,10 @@ import ( "os" "path/filepath" + libfpg "github.com/nabbar/golib/file/progress" + "github.com/nabbar/golib/archive" liberr "github.com/nabbar/golib/errors" - "github.com/nabbar/golib/ioutils" "github.com/nutsdb/nutsdb" ) @@ -84,39 +85,32 @@ func (s *snap) Prepare(opt Options, db *nutsdb.DB) liberr.Error { func (s *snap) Save(opt Options, writer io.Writer) liberr.Error { var ( - tar string - err error + e error - t ioutils.FileProgress - e liberr.Error + tmp libfpg.Progress + err liberr.Error ) - if tar, e = opt.NewTempFile("tar"); e != nil { - return e - } - defer func() { - _ = os.Remove(tar) + if tmp != nil { + _ = tmp.CloseDelete() + } }() - if t, e = ioutils.NewFileProgressPathWrite(tar, false, true, 0664); e != nil { - return e + if tmp, e = libfpg.Unique(opt.GetTempDir(), opt.NewTempFilePattern("tar")); e != nil { + return ErrorFileTemp.ErrorParent(e) } - defer func() { - _ = t.Close() - }() - - if _, e = archive.CreateArchive(archive.TypeTarGzip, t, s.path, s.path); e != nil { - return ErrorFolderArchive.Error(e) + if _, err = archive.CreateArchive(archive.TypeTarGzip, tmp, s.path, s.path); err != nil { + return ErrorFolderArchive.Error(err) } - if _, err = t.Seek(0, io.SeekStart); err != nil { - return ErrorDatabaseSnapshot.ErrorParent(err) + if _, e = tmp.Seek(0, io.SeekStart); e != nil { + return ErrorDatabaseSnapshot.ErrorParent(e) } - if _, err = t.WriteTo(writer); err != nil { - return ErrorDatabaseSnapshot.ErrorParent(err) + if _, e = tmp.WriteTo(writer); e != nil { + return ErrorDatabaseSnapshot.ErrorParent(e) } return nil @@ -124,43 +118,37 @@ func (s *snap) Save(opt Options, writer io.Writer) liberr.Error { func (s *snap) Load(opt Options, reader io.Reader) liberr.Error { var ( - arc string - out string - err error + a string + o string + e error - a ioutils.FileProgress - e liberr.Error + tmp libfpg.Progress + err liberr.Error ) - if arc, e = opt.NewTempFile("tar.gz"); e != nil { - return e - } - defer func() { - _ = os.Remove(arc) + if tmp != nil { + _ = tmp.CloseDelete() + } }() - if a, e = ioutils.NewFileProgressPathWrite(arc, false, true, 0664); e != nil { - return e + if tmp, e = libfpg.Unique(opt.GetTempDir(), opt.NewTempFilePattern("tar.gz")); e != nil { + return ErrorFileTemp.ErrorParent(e) } - defer func() { - _ = a.Close() - }() - - if _, err = a.ReadFrom(reader); err != nil { - return ErrorDatabaseSnapshot.ErrorParent(err) + if _, e = tmp.ReadFrom(reader); e != nil { + return ErrorDatabaseSnapshot.ErrorParent(e) } - if out, e = opt.NewTempFolder(); e != nil { - return e + if o, err = opt.NewTempFolder(); err != nil { + return err } - if e = archive.ExtractAll(a, filepath.Base(arc), out, opt.Permission()); e != nil { - return ErrorFolderExtract.Error(e) + if err = archive.ExtractAll(tmp, filepath.Base(a), o, opt.Permission()); err != nil { + return ErrorFolderExtract.Error(err) } - s.path = out + s.path = o return nil } diff --git a/static/interface.go b/static/interface.go index db87300..65cb3dc 100644 --- a/static/interface.go +++ b/static/interface.go @@ -33,10 +33,11 @@ import ( "sync" "sync/atomic" + libfpg "github.com/nabbar/golib/file/progress" + ginsdk "github.com/gin-gonic/gin" libctx "github.com/nabbar/golib/context" liberr "github.com/nabbar/golib/errors" - libiot "github.com/nabbar/golib/ioutils" liblog "github.com/nabbar/golib/logger" montps "github.com/nabbar/golib/monitor/types" librtr "github.com/nabbar/golib/router" @@ -66,7 +67,7 @@ type Static interface { List(rootPath string) ([]string, liberr.Error) Find(pathFile string) (io.ReadCloser, liberr.Error) Info(pathFile string) (os.FileInfo, liberr.Error) - Temp(pathFile string) (libiot.FileProgress, liberr.Error) + Temp(pathFile string) (libfpg.Progress, liberr.Error) Map(func(pathFile string, inf os.FileInfo) error) liberr.Error UseTempForFileSize(size int64) diff --git a/static/pathfile.go b/static/pathfile.go index 9fdcc70..adb373c 100644 --- a/static/pathfile.go +++ b/static/pathfile.go @@ -36,6 +36,7 @@ import ( "path" liberr "github.com/nabbar/golib/errors" + libfpg "github.com/nabbar/golib/file/progress" libiot "github.com/nabbar/golib/ioutils" ) @@ -87,7 +88,7 @@ func (s *staticHandler) _listEmbed(root string) ([]fs.DirEntry, liberr.Error) { } func (s *staticHandler) _fileGet(pathFile string) (fs.FileInfo, io.ReadCloser, liberr.Error) { - if pathFile == "" { + if len(pathFile) < 1 { return nil, nil, ErrorParamEmpty.ErrorParent(fmt.Errorf("pathfile is empty")) } @@ -153,7 +154,7 @@ func (s *staticHandler) _fileBuff(pathFile string) (io.ReadCloser, liberr.Error) } } -func (s *staticHandler) _fileTemp(pathFile string) (libiot.FileProgress, liberr.Error) { +func (s *staticHandler) _fileTemp(pathFile string) (libfpg.Progress, liberr.Error) { if pathFile == "" { return nil, ErrorParamEmpty.ErrorParent(fmt.Errorf("pathfile is empty")) } @@ -161,7 +162,7 @@ func (s *staticHandler) _fileTemp(pathFile string) (libiot.FileProgress, liberr. s.m.RLock() defer s.m.RUnlock() - var tmp libiot.FileProgress + var tmp libfpg.Progress obj, err := s.c.Open(pathFile) if err != nil && errors.Is(err, fs.ErrNotExist) { @@ -174,7 +175,7 @@ func (s *staticHandler) _fileTemp(pathFile string) (libiot.FileProgress, liberr. _ = obj.Close() }() - tmp, err = libiot.NewFileProgressTemp() + tmp, err = libfpg.Temp("") if err != nil { return nil, ErrorFiletemp.ErrorParent(err) } @@ -256,7 +257,7 @@ func (s *staticHandler) Info(pathFile string) (os.FileInfo, liberr.Error) { return s._fileInfo(pathFile) } -func (s *staticHandler) Temp(pathFile string) (libiot.FileProgress, liberr.Error) { +func (s *staticHandler) Temp(pathFile string) (libfpg.Progress, liberr.Error) { return s._fileTemp(pathFile) }