zishuo/golib
1
0
Fork 0
mirror of https://github.com/nabbar/golib.git synced 2025-09-26 20:01:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Package AWS

Browse Source 
Add feature to change http timeout directly in httpcli store in aws client
  Add copy/MPUCopy function
  Update mpu to allow copy with MPU

Package Cobra
  Fix println by fmt.Fprint to os.stdout

Package Crypt
  Refactor package to use instance of crypt instead of a global crypt unique instance
  Allow to gen key/nonce
  Add io stream reader/writer instead of only buffer slice encoder decoder

Package file/bandwidth
  Add new subpackage bandwith to limit a amount of byte read/write for a duraction
  Use file/progress function increment/reset
  Allow to add custom function increment / reset

Package IOUtils
  Remove useless file error.go, tempFile.go (replaced by package file)
  Move fileDescriptor to a subpackage fileDescriptor: allow to change nb of FD for linux/windows os
  Move Buffer ReadCloser to a new subpackage: allow to add close function to buffer, and expose read / write io interface
  Add sub package multiplexer to allow send/receive multiple io stream into on io stream

Package Socket
  Rework to expose io stream instead of buffer
  Fix minor bugs

Other:
  Bump dependencies
  Update code following bump dependencies
This commit is contained in:
Nicolas JUHEL
2023-10-03 10:18:19 +02:00
parent aed9d98203
commit b0fd08792c
61 changed files with 1612 additions and 753 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
archive/archive.go
View File

@@ -143,19 +143,25 @@ func ExtractAll(src libfpg.Progress, originalName, outputPath string, defaultDir
}
if err = libbz2.GetFile(src, tmp); err == nil {
if inf, er := tmp.Stat(); er == nil {
tmp.Reset(inf.Size())
}
return ExtractAll(tmp, originalName, outputPath, defaultDirPerm)
} else if !err.IsCode(libbz2.ErrorIOCopy) {
return err
}
if err = libgzp.GetFile(src, tmp); err == nil {
if inf, er := tmp.Stat(); er == nil {
tmp.Reset(inf.Size())
}
return ExtractAll(tmp, originalName, outputPath, defaultDirPerm)
} else if !err.IsCode(libgzp.ErrorGZReader) {
return err
}
if tmp != nil {
_ = tmp.Close()
_ = tmp.CloseDelete()
}
if i, e = os.Stat(outputPath); e != nil && os.IsNotExist(e) {

47
archive/gzip/reader.go
View File

@@ -29,14 +29,23 @@ import (
gz "compress/gzip"
"io"
libfpg "github.com/nabbar/golib/file/progress"
"github.com/nabbar/golib/errors"
)
func GetFile(src io.ReadSeeker, dst io.WriteSeeker) errors.Error {
var siz = getGunZipSize(src)
if d, k := dst.(libfpg.Progress); k && siz > 0 {
d.Reset(siz)
}
if _, e := src.Seek(0, io.SeekStart); e != nil {
return ErrorFileSeek.Error(e)
} else if _, e = dst.Seek(0, io.SeekStart); e != nil {
return ErrorFileSeek.Error(e)
} else if siz > 0 {
}
r, e := gz.NewReader(src)
@@ -52,9 +61,45 @@ func GetFile(src io.ReadSeeker, dst io.WriteSeeker) errors.Error {
/* #nosec */
if _, e = io.Copy(dst, r); e != nil {
return ErrorIOCopy.Error(e)
} else if _, e := dst.Seek(0, io.SeekStart); e != nil {
} else if _, e = dst.Seek(0, io.SeekStart); e != nil {
return ErrorFileSeek.Error(e)
} else {
return nil
}
}
func getGunZipSize(src io.ReadSeeker) int64 {
if _, e := src.Seek(0, io.SeekStart); e != nil {
return 0
}
r, e := gz.NewReader(src)
if e != nil {
return 0
}
defer func() {
_ = r.Close()
}()
if s, k := src.(libfpg.Progress); k {
s.RegisterFctReset(func(size, current int64) {
})
s.RegisterFctIncrement(func(size int64) {
})
s.RegisterFctEOF(func() {
})
}
var n int64
n, e = io.Copy(io.Discard, r)
if e != nil {
return 0
}
return n
}

4
aws/interface.go
View File

@@ -30,6 +30,7 @@ import (
"net/http"
"net/url"
"sync"
"time"
sdkaws "github.com/aws/aws-sdk-go-v2/aws"
sdksv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
@@ -90,7 +91,8 @@ type AWS interface {
GetBucketName() string
SetBucketName(bucket string)
SetHTTPTimeout(dur time.Duration) error
GetHTTPTimeout() time.Duration
GetClientS3() *sdksss.Client
SetClientS3(aws *sdksss.Client)
GetClientIam() *sdkiam.Client

32
aws/model.go
View File

@@ -27,8 +27,10 @@ package aws
import (
"context"
"fmt"
"net/http"
"sync"
"time"
sdkaws "github.com/aws/aws-sdk-go-v2/aws"
sdksv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
@@ -53,6 +55,36 @@ type client struct {
h *http.Client
}
func (c *client) SetHTTPTimeout(dur time.Duration) error {
var h *http.Client
if c.h == nil {
return fmt.Errorf("missing http client")
} else {
h = &http.Client{
Transport: c.h.Transport,
CheckRedirect: c.h.CheckRedirect,
Jar: c.h.Jar,
Timeout: dur,
}
}
if cli, err := c._NewClientS3(c.x, h); err != nil {
return err
} else {
c.s = cli
}
return nil
}
func (c *client) GetHTTPTimeout() time.Duration {
if c.h != nil {
return c.h.Timeout
}
return 0
}
func (c *client) _NewClientIAM(ctx context.Context, httpClient *http.Client) (*sdkiam.Client, error) {
var (
cfg *sdkaws.Config

140
aws/multipart/copy.go Normal file
View File

@@ -0,0 +1,140 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package multipart
import (
"fmt"
"path"
sdkaws "github.com/aws/aws-sdk-go-v2/aws"
sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
sdktyp "github.com/aws/aws-sdk-go-v2/service/s3/types"
)
func (m *mpu) Copy(fromBucket, fromObject, fromVersionId string) error {
if m == nil {
return ErrInvalidInstance
}
var (
err error
cli *sdksss.Client
src string
res *sdksss.UploadPartCopyOutput
ctx = m.getContext()
obj = m.getObject()
bck = m.getBucket()
mid = m.getMultipartID()
)
if cli = m.getClient(); cli == nil {
return ErrInvalidClient
}
src = path.Join(fromBucket, fromObject)
if len(fromVersionId) > 0 {
src += "?versionID=" + fromVersionId
}
for _, p := range m.getCopyPart(fromBucket, fromObject, fromVersionId) {
res, err = cli.UploadPartCopy(ctx, &sdksss.UploadPartCopyInput{
Bucket: sdkaws.String(bck),
CopySource: sdkaws.String(src),
Key: sdkaws.String(obj),
PartNumber: m.Counter() + 1,
UploadId: sdkaws.String(mid),
CopySourceRange: sdkaws.String("bytes=" + p),
RequestPayer: sdktyp.RequestPayerRequester,
})
if err != nil {
m.callFuncOnPushPart("", err)
return err
} else if res == nil || res.CopyPartResult == nil || res.CopyPartResult.ETag == nil || len(*res.CopyPartResult.ETag) < 1 {
m.callFuncOnPushPart("", ErrInvalidResponse)
return ErrInvalidResponse
} else {
t := *res.CopyPartResult.ETag
m.callFuncOnPushPart(t, nil)
m.RegisterPart(t)
}
}
return nil
}
func (m *mpu) getCopyPart(fromBucket, fromObject, fromVersionId string) []string {
if m == nil {
return make([]string, 0)
}
var (
err error
res = make([]string, 0)
cli *sdksss.Client
hdo *sdksss.HeadObjectOutput
ctx = m.getContext()
prt = m.s.Int64() - 1
)
if cli = m.getClient(); cli == nil {
return res
}
inp := &sdksss.HeadObjectInput{
Bucket: sdkaws.String(fromBucket),
Key: sdkaws.String(fromObject),
RequestPayer: sdktyp.RequestPayerRequester,
}
if len(fromVersionId) > 0 {
inp.VersionId = sdkaws.String(fromVersionId)
}
hdo, err = cli.HeadObject(ctx, inp)
if err != nil {
return res
} else if hdo == nil || hdo.ETag == nil || len(*hdo.ETag) < 1 {
return res
} else if size := hdo.ContentLength; size < 1 {
return res
} else {
var i int64 = 0
for i < size {
j := i + prt
if j > size {
j = size
}
res = append(res, fmt.Sprintf("%d-%d", i, j))
i = j + 1
}
}
return res
}

1
aws/multipart/interface.go
View File

@@ -59,6 +59,7 @@ type MultiPart interface {
StartMPU() error
StopMPU(abort bool) error
Copy(fromBucket, fromObject, fromVersionId string) error
AddPart(r io.Reader) (n int64, e error)
SendPart() error
CurrentSizePart() int64

5
aws/object/interface.go
View File

@@ -64,12 +64,14 @@ type Object interface {
Get(object string) (*sdksss.GetObjectOutput, error)
Put(object string, body io.Reader) error
Copy(source, destination string) error
CopyBucket(bucketSource, source, bucketDestination, destination string) error
Delete(check bool, object string) error
DeleteAll(objects *sdktps.Delete) ([]sdktps.DeletedObject, error)
GetAttributes(object, version string) (*sdksss.GetObjectAttributesOutput, error)
MultipartList(keyMarker, markerId string) (uploads []sdktps.MultipartUpload, nextKeyMarker string, nextIdMarker string, count int64, e error)
MultipartNew(partSize libsiz.Size, object string) libmpu.MultiPart
MultipartNew(partSize libsiz.Size, bucket, object string) libmpu.MultiPart
MultipartCopy(partSize libsiz.Size, bucketSource, source, version, bucketDestination, destination string) error
MultipartPut(object string, body io.Reader) error
MultipartPutCustom(partSize libsiz.Size, object string, body io.Reader) error
MultipartCancel(uploadId, key string) error
@@ -86,6 +88,7 @@ type Object interface {
VersionSize(object, version string) (size int64, err error)
VersionDelete(check bool, object, version string) error
VersionCopy(source, version, destination string) error
VersionCopyBucket(bucketSource, source, version, bucketDestination, destination string) error
VersionDeleteLock(check bool, object, version string, byPassGovernance bool) error
GetRetention(object, version string) (until time.Time, mode string, err error)

38
aws/object/multipart.go
View File

@@ -26,6 +26,7 @@
package object
import (
"fmt"
"io"
sdkaws "github.com/aws/aws-sdk-go-v2/aws"
@@ -60,8 +61,12 @@ func (cli *client) MultipartList(keyMarker, markerId string) (uploads []sdktyp.M
}
}
func (cli *client) MultipartNew(partSize libsiz.Size, object string) libmpu.MultiPart {
m := libmpu.New(partSize, object, cli.GetBucketName())
func (cli *client) MultipartNew(partSize libsiz.Size, bucket, object string) libmpu.MultiPart {
if len(bucket) < 1 {
bucket = cli.GetBucketName()
}
m := libmpu.New(partSize, object, bucket)
m.RegisterContext(cli.GetContext)
m.RegisterClientS3(func() *sdksss.Client {
return cli.s3
@@ -77,7 +82,7 @@ func (cli *client) MultipartPut(object string, body io.Reader) error {
func (cli *client) MultipartPutCustom(partSize libsiz.Size, object string, body io.Reader) error {
var (
e error
m = cli.MultipartNew(partSize, object)
m = cli.MultipartNew(partSize, "", object)
)
defer func() {
@@ -99,6 +104,33 @@ func (cli *client) MultipartPutCustom(partSize libsiz.Size, object string, body
return nil
}
func (cli *client) MultipartCopy(partSize libsiz.Size, bucketSource, source, version, bucketDestination, destination string) error {
var (
e error
m = cli.MultipartNew(partSize, bucketDestination, destination)
)
defer func() {
if m != nil {
_ = m.Close()
}
}()
if e = m.StartMPU(); e != nil {
return cli.GetError(e)
} else if e = m.Copy(bucketSource, source, version); e != nil {
return cli.GetError(e)
} else if m.Counter() < 1 {
return cli.GetError(fmt.Errorf("empty mpu copy"))
} else if e = m.StopMPU(false); e != nil {
return cli.GetError(e)
} else {
m = nil
}
return nil
}
func (cli *client) MultipartCancel(uploadId, key string) error {
res, err := cli.s3.AbortMultipartUpload(cli.GetContext(), &sdksss.AbortMultipartUploadInput{
Bucket: sdkaws.String(cli.GetBucketName()),

4
aws/object/object.go
View File

@@ -132,6 +132,10 @@ func (cli *client) Copy(source, destination string) error {
return cli.VersionCopy(source, "", destination)
}
func (cli *client) CopyBucket(bucketSource, source, bucketDestination, destination string) error {
return cli.VersionCopyBucket(bucketSource, source, "", bucketDestination, destination)
}
func (cli *client) Put(object string, body io.Reader) error {
var tpe *string

10
aws/object/version.go
View File

@@ -222,15 +222,19 @@ func (cli *client) VersionDeleteLock(check bool, object, version string, byPassG
}
func (cli *client) VersionCopy(source, version, destination string) error {
return cli.VersionCopyBucket(*cli.GetBucketAws(), source, version, *cli.GetBucketAws(), destination)
}
func (cli *client) VersionCopyBucket(bucketSource, source, version, bucketDestination, destination string) error {
in := sdksss.CopyObjectInput{
Bucket: cli.GetBucketAws(),
Bucket: sdkaws.String(bucketDestination),
Key: sdkaws.String(destination),
}
if version != "" {
in.CopySource = sdkaws.String(path.Join(*(cli.GetBucketAws()), source) + "?versionId=" + version)
in.CopySource = sdkaws.String(path.Join(bucketSource, source) + "?versionId=" + version)
} else {
in.CopySource = sdkaws.String(path.Join(*(cli.GetBucketAws()), source))
in.CopySource = sdkaws.String(path.Join(bucketSource, source))
}
_, err := cli.s3.CopyObject(cli.GetContext(), &in)

2
cobra/model.go
View File

@@ -70,7 +70,7 @@ func (c *cobra) Init() {
func (c *cobra) printHeader() {
if !c.b {
println(c.s.GetHeader())
_, _ = fmt.Fprintln(os.Stdout, c.s.GetHeader())
}
}

120
crypt/crypt.go
View File

@@ -1,120 +0,0 @@
/*
* MIT License
*
* Copyright (c) 2019 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package crypt
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/hex"
"io"
"github.com/nabbar/golib/errors"
)
var (
cryptKey = make([]byte, 32)
cryptNonce = make([]byte, 12)
)
func SetKeyHex(key, nonce string) errors.Error {
var err error
// Load your secret key from a safe place and reuse it across multiple
// Seal/Open calls. (Obviously don't use this example key for anything
// real.) If you want to convert a passphrase to a key, use a suitable
// package like bcrypt or scrypt.
cryptKey, err = hex.DecodeString(key)
if err != nil {
return ErrorHexaKey.Error(err)
}
cryptNonce, err = hex.DecodeString(nonce)
if err != nil {
return ErrorHexaNonce.Error(err)
}
return nil
}
func SetKeyByte(key [32]byte, nonce [12]byte) {
cryptKey = key[:]
cryptNonce = nonce[:]
}
func GenKeyByte() ([]byte, []byte, errors.Error) {
// Never use more than 2^32 random key with a given key because of the risk of a repeat.
if _, err := io.ReadFull(rand.Reader, cryptKey); err != nil {
return make([]byte, 32), make([]byte, 12), ErrorByteKeygen.Error(err)
}
// Never use more than 2^32 random nonces with a given key because of the risk of a repeat.
if _, err := io.ReadFull(rand.Reader, cryptNonce); err != nil {
return make([]byte, 32), make([]byte, 12), ErrorByteNonceGen.Error(err)
}
return cryptKey, cryptNonce, nil
}
func Encrypt(clearValue []byte) (string, errors.Error) {
// When decoded the key should be 16 bytes (AES-128) or 32 (AES-256).
block, err := aes.NewCipher(cryptKey)
if err != nil {
return "", ErrorAESBlock.Error(err)
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return "", ErrorAESGCM.Error(err)
}
return hex.EncodeToString(aesgcm.Seal(nil, cryptNonce, clearValue, nil)), nil
}
func Decrypt(hexaVal string) ([]byte, errors.Error) {
// When decoded the key should be 16 bytes (AES-128) or 32 (AES-256).
ciphertext, err := hex.DecodeString(hexaVal)
if err != nil {
return nil, ErrorHexaDecode.Error(err)
}
block, err := aes.NewCipher(cryptKey)
if err != nil {
return nil, ErrorAESBlock.Error(err)
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, ErrorAESGCM.Error(err)
}
if res, err := aesgcm.Open(nil, cryptNonce, ciphertext, nil); err != nil {
return res, ErrorAESDecrypt.Error(err)
} else {
return res, nil
}
}

134
crypt/interface.go Normal file
View File

@@ -0,0 +1,134 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package crypt
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/hex"
"io"
)
type Crypt interface {
Encode(p []byte) []byte
Reader(r io.Reader) io.Reader
EncodeHex(p []byte) []byte
ReaderHex(r io.Reader) io.Reader
Decode(p []byte) ([]byte, error)
Writer(w io.Writer) io.Writer
DecodeHex(p []byte) ([]byte, error)
WriterHex(w io.Writer) io.Writer
}
func GetHexKey(s string) ([32]byte, error) {
var (
err error
dst = make([]byte, 0)
key [32]byte
)
if dst, err = hex.DecodeString(s); err != nil {
return key, err
}
copy(key[:], dst[:32])
return key, nil
}
func GenKey() ([32]byte, error) {
var (
slc = make([]byte, 32)
key [32]byte
)
_, err := io.ReadFull(rand.Reader, slc)
if err != nil {
return key, err
}
copy(key[:], slc[:32])
return key, nil
}
func GetHexNonce(s string) ([12]byte, error) {
var (
err error
dst = make([]byte, 0)
non [12]byte
)
if dst, err = hex.DecodeString(s); err != nil {
return non, err
}
copy(non[:], dst[:12])
return non, nil
}
func GenNonce() ([12]byte, error) {
var (
slc = make([]byte, 32)
non [12]byte
)
_, err := io.ReadFull(rand.Reader, slc)
if err != nil {
return non, err
}
copy(non[:], slc[:12])
return non, nil
}
func New(key [32]byte, nonce [12]byte) (Crypt, error) {
var (
k = make([]byte, 32)
n = make([]byte, 12)
)
copy(k[:], key[:])
copy(n[:], nonce[:])
// When decoded the key should be 16 bytes (AES-128) or 32 (AES-256).
blk, err := aes.NewCipher(k)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(blk)
if err != nil {
return nil, err
}
return &crt{
a: gcm,
n: n,
}, nil
}

81
crypt/error.go → crypt/model.go
View File

@@ -1,7 +1,7 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -26,54 +26,43 @@
package crypt
import errors "github.com/nabbar/golib/errors"
const (
ErrorParamsEmpty errors.CodeError = iota + errors.MinPkgCrypt
ErrorHexaDecode
ErrorHexaKey
ErrorHexaNonce
ErrorByteKeygen
ErrorByteNonceGen
ErrorAESBlock
ErrorAESGCM
ErrorAESDecrypt
import (
"crypto/cipher"
"encoding/hex"
)
var isCodeError = false
func IsCodeError() bool {
return isCodeError
type crt struct {
a cipher.AEAD
n []byte
}
func init() {
isCodeError = errors.ExistInMapMessage(ErrorParamsEmpty)
errors.RegisterIdFctMessage(ErrorParamsEmpty, getMessage)
}
func getMessage(code errors.CodeError) (message string) {
switch code {
case errors.UNK_ERROR:
return ""
case ErrorParamsEmpty:
return "given parameters is empty"
case ErrorHexaDecode:
return "hexa decode error"
case ErrorHexaKey:
return "converting hexa key error"
case ErrorHexaNonce:
return "converting hexa nonce error"
case ErrorByteKeygen:
return "key generate error"
case ErrorByteNonceGen:
return "nonce generate error"
case ErrorAESBlock:
return "init AES block error"
case ErrorAESGCM:
return "init AES GCM error"
case ErrorAESDecrypt:
return "decrypt AES GCM error"
func (o *crt) Encode(p []byte) []byte {
if len(p) < 1 {
return make([]byte, 0)
}
return o.a.Seal(nil, o.n, p, nil)
}
func (o *crt) EncodeHex(p []byte) []byte {
if len(p) < 1 {
return make([]byte, 0)
}
return []byte(hex.EncodeToString(o.Encode(p)))
}
func (o *crt) Decode(p []byte) ([]byte, error) {
if len(p) < 1 {
return make([]byte, 0), nil
}
return o.a.Open(nil, o.n, p, nil)
}
func (o *crt) DecodeHex(p []byte) ([]byte, error) {
if len(p) < 1 {
return make([]byte, 0), nil
} else if dec, err := hex.DecodeString(string(p)); err != nil {
return nil, err
} else {
return o.Decode(dec)
}
return ""
}

94
crypt/reader.go Normal file
View File

@@ -0,0 +1,94 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package crypt
import (
"fmt"
"io"
)
type reader struct {
f func(p []byte) (n int, err error)
}
func (r *reader) Read(p []byte) (n int, err error) {
if r.f == nil {
return 0, fmt.Errorf("invalid reader")
} else {
return r.f(p)
}
}
func (o *crt) Reader(r io.Reader) io.Reader {
fct := func(p []byte) (n int, err error) {
var (
a = make([]byte, 0, cap(p))
b = make([]byte, cap(p)+o.a.Overhead())
)
if n, err = r.Read(b); err != nil {
return 0, err
} else if a, err = o.Decode(b[:n]); err != nil {
return 0, err
} else {
copy(p, a)
n = len(a)
clear(a)
clear(b)
return n, nil
}
}
return &reader{
f: fct,
}
}
func (o *crt) ReaderHex(r io.Reader) io.Reader {
fct := func(p []byte) (n int, err error) {
var (
a = make([]byte, 0, cap(p))
b = make([]byte, (cap(p)+o.a.Overhead())*2)
)
if n, err = r.Read(b); err != nil {
return 0, err
} else if a, err = o.DecodeHex(b[:n]); err != nil {
return 0, err
} else {
copy(p, a)
n = len(a)
clear(a)
clear(b)
return n, nil
}
}
return &reader{
f: fct,
}
}

74
crypt/writer.go Normal file
View File

@@ -0,0 +1,74 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package crypt
import (
"fmt"
"io"
)
type writer struct {
f func(p []byte) (n int, err error)
}
func (r *writer) Write(p []byte) (n int, err error) {
if r.f == nil {
return 0, fmt.Errorf("invalid reader")
} else {
return r.f(p)
}
}
func (o *crt) Writer(w io.Writer) io.Writer {
fct := func(p []byte) (n int, err error) {
n = len(p)
if _, err = w.Write(o.Encode(p)); err != nil {
return 0, err
} else {
return n, nil
}
}
return &writer{
f: fct,
}
}
func (o *crt) WriterHex(w io.Writer) io.Writer {
fct := func(p []byte) (n int, err error) {
n = len(p)
if _, err = w.Write(o.EncodeHex(p)); err != nil {
return 0, err
} else {
return n, nil
}
}
return &writer{
f: fct,
}
}

46
file/bandwidth/interface.go Normal file
View File

@@ -0,0 +1,46 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package bandwidth
import (
"sync/atomic"
libfpg "github.com/nabbar/golib/file/progress"
libsiz "github.com/nabbar/golib/size"
)
type BandWidth interface {
RegisterIncrement(fpg libfpg.Progress, fi libfpg.FctIncrement)
RegisterReset(fpg libfpg.Progress, fr libfpg.FctReset)
}
func New(bytesBySecond libsiz.Size) BandWidth {
return &bw{
t: new(atomic.Value),
l: bytesBySecond,
}
}

96
file/bandwidth/model.go Normal file
View File

@@ -0,0 +1,96 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package bandwidth
import (
"sync/atomic"
"time"
libfpg "github.com/nabbar/golib/file/progress"
libsiz "github.com/nabbar/golib/size"
)
type bw struct {
t *atomic.Value
l libsiz.Size
}
func (o *bw) RegisterIncrement(fpg libfpg.Progress, fi libfpg.FctIncrement) {
fpg.RegisterFctIncrement(func(size int64) {
o.Increment(size)
if fi != nil {
fi(size)
}
})
}
func (o *bw) RegisterReset(fpg libfpg.Progress, fr libfpg.FctReset) {
fpg.RegisterFctReset(func(size, current int64) {
o.Reset(size, current)
if fr != nil {
fr(size, current)
}
})
}
func (o *bw) Increment(size int64) {
if o == nil {
return
}
var (
i any
t time.Time
k bool
)
i = o.t.Load()
if i == nil {
t = time.Time{}
} else if t, k = i.(time.Time); !k {
t = time.Time{}
}
if !t.IsZero() && o.l > 0 {
ts := time.Since(t)
rt := float64(size) / ts.Seconds()
if lm := o.l.Float64(); rt > lm {
wt := time.Duration((rt / lm) * float64(time.Second))
if wt.Seconds() > float64(time.Second) {
time.Sleep(time.Second)
} else {
time.Sleep(wt)
}
}
}
o.t.Store(time.Now())
}
func (o *bw) Reset(size, current int64) {
o.t.Store(time.Time{})
}

6
file/progress/iobyte.go
View File

@@ -33,7 +33,7 @@ import (
func (o *progress) ReadByte() (byte, error) {
var (
p = make([]byte, 0, 1)
p = make([]byte, 1)
i int64
n int
e error
@@ -47,6 +47,8 @@ func (o *progress) ReadByte() (byte, error) {
if _, e = o.fos.Seek(i+1, io.SeekStart); e != nil && !errors.Is(e, io.EOF) {
return 0, e
}
} else if n == 0 {
return 0, e
}
return p[0], nil
@@ -54,7 +56,7 @@ func (o *progress) ReadByte() (byte, error) {
func (o *progress) WriteByte(c byte) error {
var (
p = append(make([]byte, 0, 1), c)
p = []byte{0: c}
i int64
n int
e error

12
file/progress/ioseeker.go
View File

@@ -27,12 +27,20 @@
package progress
func (o *progress) Seek(offset int64, whence int) (int64, error) {
n, err := o.seek(offset, whence)
if err != nil {
o.reset()
}
return n, err
}
func (o *progress) seek(offset int64, whence int) (int64, error) {
if o == nil || o.fos == nil {
return 0, ErrorNilPointer.Error(nil)
}
n, err := o.fos.Seek(offset, whence)
o.reset()
return n, err
}

8
file/progress/model.go
View File

@@ -83,7 +83,7 @@ func (o *progress) SizeBOF() (size int64, err error) {
return 0, ErrorNilPointer.Error(nil)
}
return o.Seek(0, io.SeekCurrent)
return o.seek(0, io.SeekCurrent)
}
func (o *progress) SizeEOF() (size int64, err error) {
@@ -97,11 +97,11 @@ func (o *progress) SizeEOF() (size int64, err error) {
b int64 // eof
)
if a, e = o.Seek(0, io.SeekCurrent); e != nil {
if a, e = o.seek(0, io.SeekCurrent); e != nil {
return 0, e
} else if b, e = o.Seek(0, io.SeekEnd); e != nil {
} else if b, e = o.seek(0, io.SeekEnd); e != nil {
return 0, e
} else if _, e = o.Seek(a, io.SeekStart); e != nil {
} else if _, e = o.seek(a, io.SeekStart); e != nil {
return 0, e
} else {
return b - a, nil

95
go.mod
View File

@@ -2,41 +2,41 @@ module github.com/nabbar/golib
go 1.21
toolchain go1.21.0
toolchain go1.21.1
require (
github.com/aws/aws-sdk-go-v2 v1.21.0
github.com/aws/aws-sdk-go-v2/config v1.18.37
github.com/aws/aws-sdk-go-v2/credentials v1.13.35
github.com/aws/aws-sdk-go-v2/config v1.18.42
github.com/aws/aws-sdk-go-v2/credentials v1.13.40
github.com/aws/aws-sdk-go-v2/service/iam v1.22.5
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
github.com/bits-and-blooms/bitset v1.8.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.0
github.com/bits-and-blooms/bitset v1.9.0
github.com/c-bata/go-prompt v0.2.6
github.com/fatih/color v1.15.0
github.com/fsnotify/fsnotify v1.6.0
github.com/fxamacker/cbor/v2 v2.5.0
github.com/gin-gonic/gin v1.9.1
github.com/go-ldap/ldap/v3 v3.4.5
github.com/go-playground/validator/v10 v10.15.2
github.com/go-ldap/ldap/v3 v3.4.6
github.com/go-playground/validator/v10 v10.15.4
github.com/google/go-github/v33 v33.0.0
github.com/hashicorp/go-hclog v1.5.0
github.com/hashicorp/go-retryablehttp v0.7.4
github.com/hashicorp/go-uuid v1.0.3
github.com/hashicorp/go-version v1.6.0
github.com/jlaffaye/ftp v0.2.0
github.com/lni/dragonboat/v3 v3.3.7
github.com/lni/dragonboat/v3 v3.3.8
github.com/matcornic/hermes/v2 v2.1.0
github.com/mattn/go-colorable v0.1.13
github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/mapstructure v1.5.0
github.com/nats-io/jwt/v2 v2.5.0
github.com/nats-io/nats-server/v2 v2.9.21
github.com/nats-io/nats.go v1.28.0
github.com/nutsdb/nutsdb v0.13.1
github.com/onsi/ginkgo/v2 v2.12.0
github.com/onsi/gomega v1.27.10
github.com/nats-io/jwt/v2 v2.5.2
github.com/nats-io/nats-server/v2 v2.10.1
github.com/nats-io/nats.go v1.30.2
github.com/nutsdb/nutsdb v0.14.1
github.com/onsi/ginkgo/v2 v2.12.1
github.com/onsi/gomega v1.28.0
github.com/pelletier/go-toml v1.9.5
github.com/prometheus/client_golang v1.16.0
github.com/prometheus/client_golang v1.17.0
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.7.0
@@ -44,15 +44,16 @@ require (
github.com/spf13/viper v1.16.0
github.com/ugorji/go/codec v1.2.11
github.com/vbauerster/mpb/v5 v5.4.0
github.com/xanzy/go-gitlab v0.90.0
github.com/vbauerster/mpb/v8 v8.6.1
github.com/xanzy/go-gitlab v0.92.3
github.com/xhit/go-simple-mail v2.2.2+incompatible
github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0
golang.org/x/net v0.14.0
golang.org/x/oauth2 v0.11.0
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
golang.org/x/net v0.15.0
golang.org/x/oauth2 v0.12.0
golang.org/x/sync v0.3.0
golang.org/x/sys v0.11.0
golang.org/x/term v0.11.0
golang.org/x/sys v0.12.0
golang.org/x/term v0.12.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/clickhouse v0.5.1
gorm.io/driver/mysql v1.5.1
@@ -65,7 +66,7 @@ require (
require (
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/ClickHouse/ch-go v0.58.2 // indirect
github.com/ClickHouse/clickhouse-go/v2 v2.13.3 // indirect
github.com/ClickHouse/clickhouse-go/v2 v2.14.1 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/sprig v2.22.0+incompatible // indirect
@@ -75,24 +76,26 @@ require (
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/antlabs/stl v0.0.1 // indirect
github.com/antlabs/timer v0.0.11 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.14.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 // indirect
github.com/aws/smithy-go v1.14.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bwmarrin/snowflake v0.3.0 // indirect
github.com/bytedance/sonic v1.10.0 // indirect
github.com/bytedance/sonic v1.10.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
github.com/chenzhuoyu/iasm v0.9.0 // indirect
@@ -102,9 +105,9 @@ require (
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/getsentry/sentry-go v0.23.0 // indirect
github.com/getsentry/sentry-go v0.24.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.6.1 // indirect
github.com/go-logr/logr v1.2.4 // indirect
@@ -131,7 +134,7 @@ require (
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.5 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/memberlist v0.5.0 // indirect
@@ -146,7 +149,7 @@ require (
github.com/jinzhu/now v1.1.5 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/juju/ratelimit v1.0.2 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
@@ -158,36 +161,36 @@ require (
github.com/mattn/go-sqlite3 v1.14.17 // indirect
github.com/mattn/go-tty v0.0.5 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/microsoft/go-mssqldb v1.5.0 // indirect
github.com/miekg/dns v1.1.55 // indirect
github.com/microsoft/go-mssqldb v1.6.0 // indirect
github.com/miekg/dns v1.1.56 // indirect
github.com/minio/highwayhash v1.0.2 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nats-io/nkeys v0.4.4 // indirect
github.com/nats-io/nkeys v0.4.5 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/paulmach/orb v0.10.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.9 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/term v1.2.0-beta.2 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/afero v1.10.0 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tidwall/btree v1.6.0 // indirect
github.com/tidwall/btree v1.7.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/valyala/fastrand v1.1.0 // indirect
github.com/valyala/histogram v1.2.0 // indirect
@@ -196,15 +199,15 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/xujiajun/mmap-go v1.0.1 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opentelemetry.io/otel v1.17.0 // indirect
go.opentelemetry.io/otel/trace v1.17.0 // indirect
golang.org/x/arch v0.4.0 // indirect
golang.org/x/crypto v0.12.0 // indirect
go.opentelemetry.io/otel v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
golang.org/x/arch v0.5.0 // indirect
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.12.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
golang.org/x/tools v0.13.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)

39
ioutils/tempFile.go → ioutils/bufferReadCloser/interface.go
View File

@@ -23,40 +23,21 @@
*
*/
package ioutils
package bufferReadCloser
import (
"os"
"path/filepath"
liberr "github.com/nabbar/golib/errors"
"bytes"
"io"
)
func NewTempFile() (*os.File, error) {
f, e := os.CreateTemp(os.TempDir(), "")
return f, ErrorIOFileTempNew.IfError(e)
type Buffer interface {
io.Reader
io.Writer
io.Closer
}
func GetTempFilePath(f *os.File) string {
if f == nil {
return ""
func New(b *bytes.Buffer) Buffer {
return &buf{
b: b,
}
return filepath.Join(os.TempDir(), filepath.Base(f.Name()))
}
func DelTempFile(f *os.File) error {
if f == nil {
return nil
}
n := GetTempFilePath(f)
a := f.Close()
e1 := ErrorIOFileTempClose.IfError(a)
b := os.Remove(n)
e2 := ErrorIOFileTempRemove.IfError(b)
return liberr.MakeIfError(e2, e1)
}

23
ioutils/bufferReadCloser.go → ioutils/bufferReadCloser/model.go
View File

@@ -23,28 +23,23 @@
*
*/
package ioutils
package bufferReadCloser
import (
"bytes"
"io"
)
import "bytes"
type brc struct {
type buf struct {
b *bytes.Buffer
}
func NewBufferReadCloser(b *bytes.Buffer) io.ReadCloser {
return &brc{
b: b,
}
}
func (b *brc) Read(p []byte) (n int, err error) {
func (b *buf) Read(p []byte) (n int, err error) {
return b.b.Read(p)
}
func (b *brc) Close() error {
func (b *buf) Write(p []byte) (n int, err error) {
return b.b.Write(p)
}
func (b *buf) Close() error {
b.b.Reset()
return nil
}

57
ioutils/encrypt/dec.go Normal file
View File

@@ -0,0 +1,57 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package encrypt
import (
"io"
libcrp "github.com/nabbar/golib/crypt"
)
type dec struct {
c libcrp.Crypt
h bool
r io.Reader
}
func (o *dec) Read(p []byte) (n int, err error) {
var (
crp = make([]byte, cap(p)*2)
res = make([]byte, cap(p))
)
if n, err = o.r.Read(crp); err != nil {
return 0, err
} else if o.h {
res, err = o.c.DecodeHex(crp)
} else {
res, err = o.c.Decode(crp)
}
copy(p, res)
return len(p), err
}

58
ioutils/encrypt/enc.go Normal file
View File

@@ -0,0 +1,58 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package encrypt
import (
"io"
libcrp "github.com/nabbar/golib/crypt"
)
type enc struct {
c libcrp.Crypt
h bool
w io.Writer
}
func (o *enc) Write(p []byte) (n int, err error) {
var (
crp []byte
siz = len(p)
)
if o.h {
crp = o.c.EncodeHex(p)
} else {
crp = o.c.Encode(p)
}
if _, err = o.w.Write(crp); err != nil {
return 0, err
} else {
return siz, err
}
}

65
ioutils/encrypt/interface.go Normal file
View File

@@ -0,0 +1,65 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package encrypt
import (
"io"
libcrp "github.com/nabbar/golib/crypt"
)
type Encrypt interface {
io.Writer
}
type Decrypt interface {
io.Reader
}
func NewEncrypt(w io.Writer, hex bool, key [32]byte, nonce [12]byte) (Encrypt, error) {
if crp, err := libcrp.New(key, nonce); err != nil {
return nil, err
} else {
return &enc{
c: crp,
h: hex,
w: w,
}, nil
}
}
func NewDecrypt(r io.Reader, hex bool, key [32]byte, nonce [12]byte) (Decrypt, error) {
if crp, err := libcrp.New(key, nonce); err != nil {
return nil, err
} else {
return &dec{
c: crp,
h: hex,
r: r,
}, nil
}
}

86
ioutils/error.go
View File

@@ -1,86 +0,0 @@
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package ioutils
import (
"fmt"
liberr "github.com/nabbar/golib/errors"
)
const (
ErrorParamEmpty liberr.CodeError = iota + liberr.MinPkgIOUtils
ErrorSyscallRLimitGet
ErrorSyscallRLimitSet
ErrorIOFileStat
ErrorIOFileSeek
ErrorIOFileTruncate
ErrorIOFileSync
ErrorIOFileOpen
ErrorIOFileTempNew
ErrorIOFileTempClose
ErrorIOFileTempRemove
ErrorNilPointer
)
func init() {
if liberr.ExistInMapMessage(ErrorParamEmpty) {
panic(fmt.Errorf("error code collision with package golib/ioutils"))
}
liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage)
}
func getMessage(code liberr.CodeError) (message string) {
switch code {
case ErrorParamEmpty:
return "given parameters is empty"
case ErrorSyscallRLimitGet:
return "error on retrieve value in syscall rlimit"
case ErrorSyscallRLimitSet:
return "error on changing value in syscall rlimit"
case ErrorIOFileStat:
return "error occur while trying to get stat of file"
case ErrorIOFileSeek:
return "error occur while trying seek into file"
case ErrorIOFileTruncate:
return "error occur while trying truncate file"
case ErrorIOFileSync:
return "error occur while trying to sync file"
case ErrorIOFileOpen:
return "error occur while trying to open file"
case ErrorIOFileTempNew:
return "error occur while trying to create new temporary file"
case ErrorIOFileTempClose:
return "closing temporary file occurs error"
case ErrorIOFileTempRemove:
return "error occurs on removing temporary file"
case ErrorNilPointer:
return "cannot call function for a nil pointer"
}
return liberr.NullMessage
}

6
ioutils/fileDescriptor.go → ioutils/fileDescriptor/fileDescriptor.go
View File

@@ -23,9 +23,7 @@
*
*/
package ioutils
import liberr "github.com/nabbar/golib/errors"
package fileDescriptor
// SystemFileDescriptor is returning current Limit & max system limit for file descriptor (open file or I/O resource) currently set in the system
// This function return the current setting (current number of file descriptor and the max value) if the newValue given is zero
@@ -48,6 +46,6 @@ import liberr "github.com/nabbar/golib/errors"
// CC=/usr/bin/x86_64-w64-mingw32-gcc CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -a -v ...
//
// Normally no problem will be result in the build.
func SystemFileDescriptor(newValue int) (current int, max int, err liberr.Error) {
func SystemFileDescriptor(newValue int) (current int, max int, err error) {
return systemFileDescriptor(newValue)
}

5
ioutils/fileDescriptor_ko.go → ioutils/fileDescriptor/fileDescriptor_ko.go
View File

@@ -26,10 +26,9 @@
*
*/
package ioutils
package fileDescriptor
import (
. "github.com/nabbar/golib/errors"
"github.com/nabbar/golib/ioutils/maxstdio"
)
@@ -39,7 +38,7 @@ const (
winHardLimitMaxStdio = 8192
)
func systemFileDescriptor(newValue int) (current int, max int, err Error) {
func systemFileDescriptor(newValue int) (current int, max int, err error) {
rLimit := maxstdio.GetMaxStdio()
if rLimit < 0 {

21
ioutils/fileDescriptor_ok.go → ioutils/fileDescriptor/fileDescriptor_ok.go
View File

@@ -26,23 +26,17 @@
*
*/
package ioutils
package fileDescriptor
import (
"syscall"
. "github.com/nabbar/golib/errors"
)
func systemFileDescriptor(newValue int) (current int, max int, err Error) {
var (
rLimit syscall.Rlimit
e error
)
func systemFileDescriptor(newValue int) (current int, max int, err error) {
var rLimit syscall.Rlimit
if e = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); e != nil {
err = ErrorSyscallRLimitGet.Error(e)
return
if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
return 0, 0, err
}
if newValue < 1 {
@@ -65,9 +59,8 @@ func systemFileDescriptor(newValue int) (current int, max int, err Error) {
}
if chg {
if e = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); e != nil {
err = ErrorSyscallRLimitSet.Error(e)
return
if err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
return 0, 0, err
}
return SystemFileDescriptor(0)

49
ioutils/multiplexer/interface.go Normal file
View File

@@ -0,0 +1,49 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package multiplexer
import (
"io"
"sync"
)
type FuncWrite func(p []byte) (n int, err error)
type MixStdOutErr[T comparable] interface {
io.Reader
Writer(key T) io.Writer
Add(key T, fct FuncWrite)
}
func New[T comparable](r io.Reader, w io.Writer) MixStdOutErr[T] {
return &mux[T]{
d: new(sync.Map),
r: r,
w: w,
}
}

32
ioutils/multiplexer/message.go Normal file
View File

@@ -0,0 +1,32 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package multiplexer
type Message[T comparable] struct {
Stream T `cbor:"1,keyasint"`
Message []byte `cbor:"2,keyasint"`
}

119
ioutils/multiplexer/model.go Normal file
View File

@@ -0,0 +1,119 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package multiplexer
import (
"fmt"
"io"
"sync"
"time"
libcbr "github.com/fxamacker/cbor/v2"
)
type mux[T comparable] struct {
d *sync.Map
r io.Reader
w io.Writer
}
func (o *mux[T]) Add(key T, fct FuncWrite) {
if o.r == nil {
return
}
o.d.Store(key, fct)
}
func (o *mux[T]) Read(p []byte) (n int, err error) {
if o.r == nil {
return 0, fmt.Errorf("invalid stream io reader")
}
var (
d = libcbr.NewDecoder(o.r)
c T
m Message[T]
)
if err = d.Decode(&m); err != nil {
return 0, err
} else if m.Stream == c {
return 0, fmt.Errorf("invalid stream key '%s'", m.Stream)
} else if len(m.Message) < 1 {
return 0, nil
} else if i, l := o.d.Load(m.Stream); !l {
return 0, fmt.Errorf("invalid read func for stream key '%s'", m.Stream)
} else if i == nil {
return 0, fmt.Errorf("invalid read func for stream key '%s'", m.Stream)
} else if f, k := i.(FuncWrite); !k {
return 0, fmt.Errorf("invalid read func for stream key '%s'", m.Stream)
} else if f == nil {
return 0, fmt.Errorf("invalid read func for stream key '%s'", m.Stream)
} else {
return f(m.Message)
}
}
func (o *mux[T]) write(key T, p []byte) (n int, err error) {
if o.w == nil {
return 0, fmt.Errorf("invalid stream io writer")
}
var (
d = libcbr.NewEncoder(o.w)
c T
m = Message[T]{
Stream: key,
Message: p,
}
)
defer func() {
time.Sleep(5 * time.Millisecond)
}()
n = len(p)
if m.Stream == c {
return 0, fmt.Errorf("invalid stream key '%s'", m.Stream)
} else if len(m.Message) < 1 {
return 0, nil
} else if err = d.Encode(m); err != nil {
return 0, err
} else {
return n, nil
}
}
func (o *mux[T]) Writer(key T) io.Writer {
return &writer[T]{
f: func(p []byte) (n int, err error) {
return o.write(key, p)
},
}
}

41
ioutils/multiplexer/writer.go Normal file
View File

@@ -0,0 +1,41 @@
/*
* MIT License
*
* Copyright (c) 2023 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package multiplexer
import "fmt"
type writer[T comparable] struct {
f func(p []byte) (n int, err error)
}
func (w *writer[T]) Write(p []byte) (n int, err error) {
if w.f == nil {
return 0, fmt.Errorf("invalid writer function")
} else {
return w.f(p)
}
}

28
nutsdb/Command.go
View File

@@ -32,7 +32,6 @@ package nutsdb
import (
"github.com/nutsdb/nutsdb"
"github.com/nutsdb/nutsdb/ds/zset"
)
type Commands interface {
@@ -72,15 +71,6 @@ type CommandBPTree interface {
// Delete removes a key from the bucket at given bucket and key.
Delete(bucket string, key []byte) error
// FindTxIDOnDisk returns if txId on disk at given fid and txID.
FindTxIDOnDisk(fID, txID uint64) (ok bool, err error)
// FindOnDisk returns entry on disk at given fID, rootOff and key.
FindOnDisk(fID uint64, rootOff uint64, key, newKey []byte) (entry *nutsdb.Entry, err error)
// FindLeafOnDisk returns binary leaf node on disk at given fId, rootOff and key.
FindLeafOnDisk(fID int64, rootOff int64, key, newKey []byte) (bn *nutsdb.BinaryNode, err error)
}
type CommandSet interface {
@@ -181,7 +171,7 @@ type CommandZSet interface {
ZAdd(bucket string, key []byte, score float64, val []byte) error
// ZMembers returns all the members of the set value stored at bucket.
ZMembers(bucket string) (map[string]*zset.SortedSetNode, error)
ZMembers(bucket string) (map[string]*nutsdb.SortedSetMember, error)
// ZCard returns the sorted set cardinality (number of elements) of the sorted set stored at bucket.
ZCard(bucket string) (int, error)
@@ -191,26 +181,26 @@ type CommandZSet interface {
// Limit int // limit the max nodes to return
// ExcludeStart bool // exclude start value, so it search in interval (start, end] or (start, end)
// ExcludeEnd bool // exclude end value, so it search in interval [start, end) or (start, end)
ZCount(bucket string, start, end float64, opts *zset.GetByScoreRangeOptions) (int, error)
ZCount(bucket string, start, end float64, opts *nutsdb.GetByScoreRangeOptions) (int, error)
// ZPopMax removes and returns the member with the highest score in the sorted set stored at bucket.
ZPopMax(bucket string) (*zset.SortedSetNode, error)
ZPopMax(bucket string) (*nutsdb.SortedSetMember, error)
// ZPopMin removes and returns the member with the lowest score in the sorted set stored at bucket.
ZPopMin(bucket string) (*zset.SortedSetNode, error)
ZPopMin(bucket string) (*nutsdb.SortedSetMember, error)
// ZPeekMax returns the member with the highest score in the sorted set stored at bucket.
ZPeekMax(bucket string) (*zset.SortedSetNode, error)
ZPeekMax(bucket string) (*nutsdb.SortedSetMember, error)
// ZPeekMin returns the member with the lowest score in the sorted set stored at bucket.
ZPeekMin(bucket string) (*zset.SortedSetNode, error)
ZPeekMin(bucket string) (*nutsdb.SortedSetMember, error)
// ZRangeByScore returns all the elements in the sorted set at bucket with a score between min and max.
ZRangeByScore(bucket string, start, end float64, opts *zset.GetByScoreRangeOptions) ([]*zset.SortedSetNode, error)
ZRangeByScore(bucket string, start, end float64, opts *nutsdb.GetByScoreRangeOptions) ([]*nutsdb.SortedSetMember, error)
// ZRangeByRank returns all the elements in the sorted set in one bucket and key
// with a rank between start and end (including elements with rank equal to start or end).
ZRangeByRank(bucket string, start, end int) ([]*zset.SortedSetNode, error)
ZRangeByRank(bucket string, start, end int) ([]*nutsdb.SortedSetMember, error)
// ZRem removes the specified members from the sorted set stored in one bucket at given bucket and key.
ZRem(bucket, key string) error
@@ -231,5 +221,5 @@ type CommandZSet interface {
ZScore(bucket string, key []byte) (float64, error)
// ZGetByKey returns node in the bucket at given bucket and key.
ZGetByKey(bucket string, key []byte) (*zset.SortedSetNode, error)
ZGetByKey(bucket string, key []byte) (*nutsdb.SortedSetMember, error)
}

121
nutsdb/client.go
View File

@@ -41,7 +41,6 @@ import (
libclu "github.com/nabbar/golib/cluster"
liberr "github.com/nabbar/golib/errors"
"github.com/nutsdb/nutsdb"
"github.com/nutsdb/nutsdb/ds/zset"
)
type Client interface {
@@ -219,7 +218,7 @@ func (c *clientNutDB) Run(cmd CmdCode, args []string) (*CommandResponse, liberr.
}
params := make([]reflect.Value, nbPrm)
opt := &zset.GetByScoreRangeOptions{
opt := &nutsdb.GetByScoreRangeOptions{
Limit: 0,
ExcludeStart: false,
ExcludeEnd: false,
@@ -510,90 +509,6 @@ func (c *clientNutDB) Delete(bucket string, key []byte) error {
return nil
}
// nolint #dupl
func (c *clientNutDB) FindTxIDOnDisk(fID, txID uint64) (ok bool, err error) {
var (
k bool
f liberr.Error
r *CommandResponse
d *CommandRequest
)
d = NewCommandByCaller(fID, txID)
if r, f = c.call(d, true); f != nil {
return false, f
} else if r == nil {
return false, nil
} else if r.Error != nil {
return false, r.Error
} else if len(r.Value) < 1 {
return false, nil
}
if ok, k = r.Value[0].(bool); !k {
ok = false
}
return
}
// nolint #dupl
func (c *clientNutDB) FindOnDisk(fID uint64, rootOff uint64, key, newKey []byte) (entry *nutsdb.Entry, err error) {
var (
k bool
f liberr.Error
r *CommandResponse
d *CommandRequest
)
d = NewCommandByCaller(fID, rootOff, key, newKey)
if r, f = c.call(d, true); f != nil {
return nil, f
} else if r == nil {
return nil, nil
} else if r.Error != nil {
return nil, r.Error
} else if len(r.Value) < 1 {
return nil, nil
}
if entry, k = r.Value[0].(*nutsdb.Entry); !k {
entry = nil
}
return
}
// nolint #dupl
func (c *clientNutDB) FindLeafOnDisk(fID int64, rootOff int64, key, newKey []byte) (bn *nutsdb.BinaryNode, err error) {
var (
k bool
f liberr.Error
r *CommandResponse
d *CommandRequest
)
d = NewCommandByCaller(fID, rootOff, key, newKey)
if r, f = c.call(d, true); f != nil {
return nil, f
} else if r == nil {
return nil, nil
} else if r.Error != nil {
return nil, r.Error
} else if len(r.Value) < 1 {
return nil, nil
}
if bn, k = r.Value[0].(*nutsdb.BinaryNode); !k {
bn = nil
}
return
}
// nolint #dupl
func (c *clientNutDB) SAdd(bucket string, key []byte, items ...[]byte) error {
var (
@@ -1274,7 +1189,7 @@ func (c *clientNutDB) ZAdd(bucket string, key []byte, score float64, val []byte)
}
// nolint #dupl
func (c *clientNutDB) ZMembers(bucket string) (list map[string]*zset.SortedSetNode, err error) {
func (c *clientNutDB) ZMembers(bucket string) (list map[string]*nutsdb.SortedSetMember, err error) {
var (
k bool
f liberr.Error
@@ -1294,7 +1209,7 @@ func (c *clientNutDB) ZMembers(bucket string) (list map[string]*zset.SortedSetNo
return nil, nil
}
if list, k = r.Value[0].(map[string]*zset.SortedSetNode); !k {
if list, k = r.Value[0].(map[string]*nutsdb.SortedSetMember); !k {
list = nil
}
@@ -1330,7 +1245,7 @@ func (c *clientNutDB) ZCard(bucket string) (card int, err error) {
}
// nolint #dupl
func (c *clientNutDB) ZCount(bucket string, start, end float64, opts *zset.GetByScoreRangeOptions) (number int, err error) {
func (c *clientNutDB) ZCount(bucket string, start, end float64, opts *nutsdb.GetByScoreRangeOptions) (number int, err error) {
var (
k bool
f liberr.Error
@@ -1358,7 +1273,7 @@ func (c *clientNutDB) ZCount(bucket string, start, end float64, opts *zset.GetBy
}
// nolint #dupl
func (c *clientNutDB) ZPopMax(bucket string) (item *zset.SortedSetNode, err error) {
func (c *clientNutDB) ZPopMax(bucket string) (item *nutsdb.SortedSetMember, err error) {
var (
k bool
f liberr.Error
@@ -1378,7 +1293,7 @@ func (c *clientNutDB) ZPopMax(bucket string) (item *zset.SortedSetNode, err erro
return nil, nil
}
if item, k = r.Value[0].(*zset.SortedSetNode); !k {
if item, k = r.Value[0].(*nutsdb.SortedSetMember); !k {
item = nil
}
@@ -1386,7 +1301,7 @@ func (c *clientNutDB) ZPopMax(bucket string) (item *zset.SortedSetNode, err erro
}
// nolint #dupl
func (c *clientNutDB) ZPopMin(bucket string) (item *zset.SortedSetNode, err error) {
func (c *clientNutDB) ZPopMin(bucket string) (item *nutsdb.SortedSetMember, err error) {
var (
k bool
f liberr.Error
@@ -1406,7 +1321,7 @@ func (c *clientNutDB) ZPopMin(bucket string) (item *zset.SortedSetNode, err erro
return nil, nil
}
if item, k = r.Value[0].(*zset.SortedSetNode); !k {
if item, k = r.Value[0].(*nutsdb.SortedSetMember); !k {
item = nil
}
@@ -1414,7 +1329,7 @@ func (c *clientNutDB) ZPopMin(bucket string) (item *zset.SortedSetNode, err erro
}
// nolint #dupl
func (c *clientNutDB) ZPeekMax(bucket string) (item *zset.SortedSetNode, err error) {
func (c *clientNutDB) ZPeekMax(bucket string) (item *nutsdb.SortedSetMember, err error) {
var (
k bool
f liberr.Error
@@ -1434,7 +1349,7 @@ func (c *clientNutDB) ZPeekMax(bucket string) (item *zset.SortedSetNode, err err
return nil, nil
}
if item, k = r.Value[0].(*zset.SortedSetNode); !k {
if item, k = r.Value[0].(*nutsdb.SortedSetMember); !k {
item = nil
}
@@ -1442,7 +1357,7 @@ func (c *clientNutDB) ZPeekMax(bucket string) (item *zset.SortedSetNode, err err
}
// nolint #dupl
func (c *clientNutDB) ZPeekMin(bucket string) (item *zset.SortedSetNode, err error) {
func (c *clientNutDB) ZPeekMin(bucket string) (item *nutsdb.SortedSetMember, err error) {
var (
k bool
f liberr.Error
@@ -1462,7 +1377,7 @@ func (c *clientNutDB) ZPeekMin(bucket string) (item *zset.SortedSetNode, err err
return nil, nil
}
if item, k = r.Value[0].(*zset.SortedSetNode); !k {
if item, k = r.Value[0].(*nutsdb.SortedSetMember); !k {
item = nil
}
@@ -1470,7 +1385,7 @@ func (c *clientNutDB) ZPeekMin(bucket string) (item *zset.SortedSetNode, err err
}
// nolint #dupl
func (c *clientNutDB) ZRangeByScore(bucket string, start, end float64, opts *zset.GetByScoreRangeOptions) (list []*zset.SortedSetNode, err error) {
func (c *clientNutDB) ZRangeByScore(bucket string, start, end float64, opts *nutsdb.GetByScoreRangeOptions) (list []*nutsdb.SortedSetMember, err error) {
var (
k bool
f liberr.Error
@@ -1490,7 +1405,7 @@ func (c *clientNutDB) ZRangeByScore(bucket string, start, end float64, opts *zse
return nil, nil
}
if list, k = r.Value[0].([]*zset.SortedSetNode); !k {
if list, k = r.Value[0].([]*nutsdb.SortedSetMember); !k {
list = nil
}
@@ -1498,7 +1413,7 @@ func (c *clientNutDB) ZRangeByScore(bucket string, start, end float64, opts *zse
}
// nolint #dupl
func (c *clientNutDB) ZRangeByRank(bucket string, start, end int) (list []*zset.SortedSetNode, err error) {
func (c *clientNutDB) ZRangeByRank(bucket string, start, end int) (list []*nutsdb.SortedSetMember, err error) {
var (
k bool
f liberr.Error
@@ -1518,7 +1433,7 @@ func (c *clientNutDB) ZRangeByRank(bucket string, start, end int) (list []*zset.
return nil, nil
}
if list, k = r.Value[0].([]*zset.SortedSetNode); !k {
if list, k = r.Value[0].([]*nutsdb.SortedSetMember); !k {
list = nil
}
@@ -1652,7 +1567,7 @@ func (c *clientNutDB) ZScore(bucket string, key []byte) (score float64, err erro
}
// nolint #dupl
func (c *clientNutDB) ZGetByKey(bucket string, key []byte) (item *zset.SortedSetNode, err error) {
func (c *clientNutDB) ZGetByKey(bucket string, key []byte) (item *nutsdb.SortedSetMember, err error) {
var (
k bool
f liberr.Error
@@ -1672,7 +1587,7 @@ func (c *clientNutDB) ZGetByKey(bucket string, key []byte) (item *zset.SortedSet
return nil, nil
}
if item, k = r.Value[0].(*zset.SortedSetNode); !k {
if item, k = r.Value[0].(*nutsdb.SortedSetMember); !k {
item = nil
}

2
nutsdb/configDb.go
View File

@@ -78,8 +78,6 @@ func (o NutsDBOptions) GetNutsDBOptions(dataDir string) nutsdb.Options {
switch o.EntryIdxMode {
case nutsdb.HintKeyAndRAMIdxMode:
d.EntryIdxMode = nutsdb.HintKeyAndRAMIdxMode
case nutsdb.HintBPTSparseIdxMode:
d.EntryIdxMode = nutsdb.HintBPTSparseIdxMode
default:
d.EntryIdxMode = nutsdb.HintKeyValAndRAMIdxMode
}

11
nutsdb/shell.go
View File

@@ -37,7 +37,6 @@ import (
shlcmd "github.com/nabbar/golib/shell/command"
"github.com/nutsdb/nutsdb"
"github.com/nutsdb/nutsdb/ds/zset"
)
type shellCommand struct {
@@ -130,21 +129,19 @@ func (s *shellCommand) parse(buf, err io.Writer, val interface{}) {
return
}
if values, ok := val.(map[string]*zset.SortedSetNode); ok {
if values, ok := val.(map[string]*nutsdb.SortedSetMember); ok {
for _, v := range values {
_, _ = fmt.Fprintf(buf, "Key: %s\n", v.Key())
_, _ = fmt.Fprintf(buf, "Val: %s\n", string(v.Value))
_, _ = fmt.Fprintf(buf, "Score: %v\n", v.Score())
_, _ = fmt.Fprintf(buf, "Score: %v\n", v.Score)
_, _ = fmt.Fprintf(buf, "\n")
}
return
}
if values, ok := val.([]*zset.SortedSetNode); ok {
if values, ok := val.([]*nutsdb.SortedSetMember); ok {
for _, v := range values {
_, _ = fmt.Fprintf(buf, "Key: %s\n", v.Key())
_, _ = fmt.Fprintf(buf, "Val: %s\n", string(v.Value))
_, _ = fmt.Fprintf(buf, "Score: %v\n", v.Score())
_, _ = fmt.Fprintf(buf, "Score: %v\n", v.Score)
_, _ = fmt.Fprintf(buf, "\n")
}
return

33
socket/client/tcp/model.go
View File

@@ -110,25 +110,22 @@ func (o *cltt) dial(ctx context.Context) (net.Conn, error) {
}
}
func (o *cltt) Do(ctx context.Context, request io.Reader) (io.Reader, error) {
func (o *cltt) Do(ctx context.Context, request io.Reader, fct libsck.Response) error {
if o == nil {
return nil, ErrInstance
return ErrInstance
}
var (
e error
lc net.Addr
rm net.Addr
e error
lc net.Addr
rm net.Addr
cnn net.Conn
buf = o.buffRead()
)
o.fctInfo(nil, nil, libsck.ConnectionDial)
if cnn, e = o.dial(ctx); e != nil {
o.fctError(e)
return nil, e
return e
}
defer o.fctError(cnn.Close())
@@ -143,24 +140,26 @@ func (o *cltt) Do(ctx context.Context, request io.Reader) (io.Reader, error) {
o.fctInfo(lc, rm, libsck.ConnectionWrite)
if _, e = io.Copy(cnn, request); e != nil {
o.fctError(e)
return nil, e
return e
}
}
o.fctInfo(lc, rm, libsck.ConnectionCloseWrite)
if e = cnn.(*net.TCPConn).CloseWrite(); e != nil {
o.fctError(e)
return nil, e
return e
}
o.fctInfo(lc, rm, libsck.ConnectionRead)
if _, e = io.Copy(buf, cnn); e != nil {
o.fctError(e)
return nil, e
o.fctInfo(lc, rm, libsck.ConnectionHandler)
if fct != nil {
fct(cnn)
}
o.fctInfo(lc, rm, libsck.ConnectionCloseRead)
o.fctError(cnn.(*net.TCPConn).CloseRead())
if e = cnn.(*net.TCPConn).CloseRead(); e != nil {
o.fctError(e)
return e
}
return buf, nil
return nil
}

24
socket/client/udp/model.go
View File

@@ -104,25 +104,22 @@ func (o *cltu) dial(ctx context.Context) (net.Conn, error) {
}
}
func (o *cltu) Do(ctx context.Context, request io.Reader) (io.Reader, error) {
func (o *cltu) Do(ctx context.Context, request io.Reader, fct libsck.Response) error {
if o == nil {
return nil, ErrInstance
return ErrInstance
}
var (
e error
lc net.Addr
rm net.Addr
e error
lc net.Addr
rm net.Addr
cnn net.Conn
buf = o.buffRead()
)
o.fctInfo(nil, nil, libsck.ConnectionDial)
if cnn, e = o.dial(ctx); e != nil {
o.fctError(e)
return nil, e
return e
}
defer o.fctError(cnn.Close())
@@ -137,9 +134,14 @@ func (o *cltu) Do(ctx context.Context, request io.Reader) (io.Reader, error) {
o.fctInfo(lc, rm, libsck.ConnectionWrite)
if _, e = io.Copy(cnn, request); e != nil {
o.fctError(e)
return nil, e
return e
}
}
return buf, nil
o.fctInfo(lc, rm, libsck.ConnectionHandler)
if fct != nil {
fct(cnn)
}
return nil
}

37
socket/client/unix/model.go
View File

@@ -113,30 +113,27 @@ func (o *cltx) dial(ctx context.Context) (net.Conn, error) {
}
}
func (o *cltx) Do(ctx context.Context, request io.Reader) (io.Reader, error) {
func (o *cltx) Do(ctx context.Context, request io.Reader, fct libsck.Response) error {
if o == nil {
return nil, ErrInvalidInstance
return ErrInvalidInstance
}
var (
e error
lc net.Addr
rm net.Addr
e error
lc net.Addr
rm net.Addr
cnn net.Conn
buf = o.buffRead()
)
o.fctInfo(nil, nil, libsck.ConnectionDial)
if cnn, e = o.dial(ctx); e != nil {
o.fctError(e)
return nil, e
return e
}
defer func() {
e := cnn.Close()
o.fctError(e)
err := cnn.Close()
o.fctError(err)
}()
lc = cnn.LocalAddr()
@@ -149,24 +146,26 @@ func (o *cltx) Do(ctx context.Context, request io.Reader) (io.Reader, error) {
o.fctInfo(lc, rm, libsck.ConnectionWrite)
if _, e = io.Copy(cnn, request); e != nil {
o.fctError(e)
return nil, e
return e
}
}
o.fctInfo(lc, rm, libsck.ConnectionCloseWrite)
if e = cnn.(*net.UnixConn).CloseWrite(); e != nil {
o.fctError(e)
return nil, e
return e
}
o.fctInfo(lc, rm, libsck.ConnectionRead)
if _, e = io.Copy(buf, cnn); e != nil {
o.fctError(e)
return nil, e
o.fctInfo(lc, rm, libsck.ConnectionHandler)
if fct != nil {
fct(cnn)
}
o.fctInfo(lc, rm, libsck.ConnectionCloseRead)
o.fctError(cnn.(*net.UnixConn).CloseRead())
if e = cnn.(*net.UnixConn).CloseRead(); e != nil {
o.fctError(e)
return e
}
return buf, nil
return nil
}

15
socket/config/server.go
View File

@@ -36,17 +36,16 @@ import (
)
type ServerConfig struct {
Network libptc.NetworkProtocol ``
Address string
PermFile os.FileMode
BuffSizeRead int32
BuffSizeWrite int32
TimeoutRead time.Duration
TimeoutWrite time.Duration
Network libptc.NetworkProtocol ``
Address string
PermFile os.FileMode
BuffSizeRead int32
TimeoutRead time.Duration
TimeoutWrite time.Duration
}
func (o ServerConfig) New(handler libsck.Handler) (libsck.Server, error) {
s, e := scksrv.New(handler, o.Network, o.BuffSizeRead, o.BuffSizeWrite, o.Address, o.PermFile)
s, e := scksrv.New(handler, o.Network, o.BuffSizeRead, o.Address, o.PermFile)
if e != nil {
s.SetReadTimeout(o.TimeoutRead)

3
socket/interface.go
View File

@@ -51,6 +51,7 @@ const (
type FuncError func(e error)
type FuncInfo func(local, remote net.Addr, state ConnState)
type Handler func(request io.Reader, response io.Writer)
type Response func(r io.Reader)
type Server interface {
RegisterFuncError(f FuncError)
@@ -68,5 +69,5 @@ type Client interface {
RegisterFuncError(f FuncError)
RegisterFuncInfo(f FuncInfo)
Do(ctx context.Context, request io.Reader) (io.Reader, error)
Do(ctx context.Context, request io.Reader, fct Response) error
}

8
socket/server/interface_linux.go
View File

@@ -42,20 +42,20 @@ import (
scksrx "github.com/nabbar/golib/socket/server/unix"
)
func New(handler libsck.Handler, proto libptc.NetworkProtocol, sizeBufferRead, sizeBufferWrite int32, address string, perm os.FileMode) (libsck.Server, error) {
func New(handler libsck.Handler, proto libptc.NetworkProtocol, sizeBufferRead int32, address string, perm os.FileMode) (libsck.Server, error) {
switch proto {
case libptc.NetworkUnix:
if strings.EqualFold(runtime.GOOS, "linux") {
s := scksrx.New(handler, sizeBufferRead, sizeBufferWrite)
s := scksrx.New(handler, sizeBufferRead)
s.RegisterSocket(address, perm)
return s, nil
}
case libptc.NetworkTCP, libptc.NetworkTCP4, libptc.NetworkTCP6:
s := scksrt.New(handler, sizeBufferRead, sizeBufferWrite)
s := scksrt.New(handler, sizeBufferRead)
e := s.RegisterServer(address)
return s, e
case libptc.NetworkUDP, libptc.NetworkUDP4, libptc.NetworkUDP6:
s := scksru.New(handler, sizeBufferRead, sizeBufferWrite)
s := scksru.New(handler, sizeBufferRead)
e := s.RegisterServer(address)
return s, e
}

6
socket/server/interface_other.go
View File

@@ -39,14 +39,14 @@ import (
scksru "github.com/nabbar/golib/socket/server/udp"
)
func New(handler libsck.Handler, proto libptc.NetworkProtocol, sizeBufferRead, sizeBufferWrite int32, address string, perm os.FileMode) (libsck.Server, error) {
func New(handler libsck.Handler, proto libptc.NetworkProtocol, sizeBufferRead int32, address string, perm os.FileMode) (libsck.Server, error) {
switch proto {
case libptc.NetworkTCP, libptc.NetworkTCP4, libptc.NetworkTCP6:
s := scksrt.New(handler, sizeBufferRead, sizeBufferWrite)
s := scksrt.New(handler, sizeBufferRead)
e := s.RegisterServer(address)
return s, e
case libptc.NetworkUDP, libptc.NetworkUDP4, libptc.NetworkUDP6:
s := scksru.New(handler, sizeBufferRead, sizeBufferWrite)
s := scksru.New(handler, sizeBufferRead)
e := s.RegisterServer(address)
return s, e
}

6
socket/server/tcp/inerface.go
View File

@@ -37,7 +37,7 @@ type ServerTcp interface {
RegisterServer(address string) error
}
func New(h libsck.Handler, sizeBuffRead, sizeBuffWrite int32) ServerTcp {
func New(h libsck.Handler, sizeBuffRead int32) ServerTcp {
c := new(atomic.Value)
c.Store(make(chan []byte))
@@ -50,9 +50,6 @@ func New(h libsck.Handler, sizeBuffRead, sizeBuffWrite int32) ServerTcp {
sr := new(atomic.Int32)
sr.Store(sizeBuffRead)
sw := new(atomic.Int32)
sw.Store(sizeBuffWrite)
return &srv{
l: nil,
h: f,
@@ -63,6 +60,5 @@ func New(h libsck.Handler, sizeBuffRead, sizeBuffWrite int32) ServerTcp {
tr: new(atomic.Value),
tw: new(atomic.Value),
sr: sr,
sw: sw,
}
}

19
socket/server/tcp/listener.go
View File

@@ -65,15 +65,6 @@ func (o *srv) buffRead() *bytes.Buffer {
return bytes.NewBuffer(make([]byte, 0, libsck.DefaultBufferSize))
}
func (o *srv) buffWrite() *bytes.Buffer {
v := o.sw.Load()
if v > 0 {
return bytes.NewBuffer(make([]byte, 0, int(v)))
}
return bytes.NewBuffer(make([]byte, 0, libsck.DefaultBufferSize))
}
func (o *srv) getAddress() *url.URL {
f := o.ad.Load()
if f != nil {
@@ -133,7 +124,6 @@ func (o *srv) Conn(conn net.Conn) {
tr = o.timeoutRead()
tw = o.timeoutWrite()
br = o.buffRead()
bw = o.buffWrite()
)
defer o.fctInfo(lc, rm, libsck.ConnectionClose)
@@ -167,14 +157,7 @@ func (o *srv) Conn(conn net.Conn) {
if h := o.handler(); h != nil {
o.fctInfo(lc, rm, libsck.ConnectionHandler)
h(br, bw)
}
if bw.Len() > 0 {
o.fctInfo(lc, rm, libsck.ConnectionWrite)
if _, e := io.Copy(conn, bw); e != nil {
o.fctError(e)
}
h(br, conn)
}
o.fctInfo(lc, rm, libsck.ConnectionCloseWrite)

1
socket/server/tcp/model.go
View File

@@ -57,7 +57,6 @@ type srv struct {
tr *atomic.Value // connection read timeout
tw *atomic.Value // connection write timeout
sr *atomic.Int32 // read buffer size
sw *atomic.Int32 // write buffer size
ad *atomic.Value // Server address url
}

6
socket/server/udp/inerface.go
View File

@@ -37,7 +37,7 @@ type ServerTcp interface {
RegisterServer(address string) error
}
func New(h libsck.Handler, sizeBuffRead, sizeBuffWrite int32) ServerTcp {
func New(h libsck.Handler, sizeBuffRead int32) ServerTcp {
c := new(atomic.Value)
c.Store(make(chan []byte))
@@ -50,9 +50,6 @@ func New(h libsck.Handler, sizeBuffRead, sizeBuffWrite int32) ServerTcp {
sr := new(atomic.Int32)
sr.Store(sizeBuffRead)
sw := new(atomic.Int32)
sw.Store(sizeBuffWrite)
return &srv{
l: nil,
h: f,
@@ -63,6 +60,5 @@ func New(h libsck.Handler, sizeBuffRead, sizeBuffWrite int32) ServerTcp {
tr: new(atomic.Value),
tw: new(atomic.Value),
sr: sr,
sw: sw,
}
}

12
socket/server/udp/listener.go
View File

@@ -65,15 +65,6 @@ func (o *srv) buffRead() *bytes.Buffer {
return bytes.NewBuffer(make([]byte, 0, libsck.DefaultBufferSize))
}
func (o *srv) buffWrite() *bytes.Buffer {
v := o.sw.Load()
if v > 0 {
return bytes.NewBuffer(make([]byte, 0, int(v)))
}
return bytes.NewBuffer(make([]byte, 0, libsck.DefaultBufferSize))
}
func (o *srv) getAddress() *url.URL {
f := o.ad.Load()
if f != nil {
@@ -133,7 +124,6 @@ func (o *srv) Conn(conn net.Conn) {
tr = o.timeoutRead()
tw = o.timeoutWrite()
br = o.buffRead()
bw = io.Discard
)
defer o.fctInfo(lc, rm, libsck.ConnectionClose)
@@ -161,6 +151,6 @@ func (o *srv) Conn(conn net.Conn) {
if h := o.handler(); h != nil {
o.fctInfo(lc, rm, libsck.ConnectionHandler)
h(br, bw)
h(br, conn)
}
}

1
socket/server/udp/model.go
View File

@@ -57,7 +57,6 @@ type srv struct {
tr *atomic.Value // connection read timeout
tw *atomic.Value // connection write timeout
sr *atomic.Int32 // read buffer size
sw *atomic.Int32 // write buffer size
ad *atomic.Value // Server address url
}

6
socket/server/unix/inerface.go
View File

@@ -41,7 +41,7 @@ type ServerUnix interface {
RegisterSocket(unixFile string, perm os.FileMode)
}
func New(h libsck.Handler, sizeBuffRead, sizeBuffWrite int32) ServerUnix {
func New(h libsck.Handler, sizeBuffRead int32) ServerUnix {
c := new(atomic.Value)
c.Store(make(chan []byte))
@@ -54,9 +54,6 @@ func New(h libsck.Handler, sizeBuffRead, sizeBuffWrite int32) ServerUnix {
sr := new(atomic.Int32)
sr.Store(sizeBuffRead)
sw := new(atomic.Int32)
sw.Store(sizeBuffWrite)
fp := new(atomic.Value)
fp.Store("")
@@ -73,7 +70,6 @@ func New(h libsck.Handler, sizeBuffRead, sizeBuffWrite int32) ServerUnix {
tr: new(atomic.Value),
tw: new(atomic.Value),
sr: sr,
sw: sw,
fs: fp,
fp: pe,
}

21
socket/server/unix/listener.go
View File

@@ -81,15 +81,6 @@ func (o *srv) buffRead() *bytes.Buffer {
return bytes.NewBuffer(make([]byte, 0, libsck.DefaultBufferSize))
}
func (o *srv) buffWrite() *bytes.Buffer {
v := o.sw.Load()
if v > 0 {
return bytes.NewBuffer(make([]byte, 0, int(v)))
}
return bytes.NewBuffer(make([]byte, 0, libsck.DefaultBufferSize))
}
func (o *srv) getSocketFile() (string, error) {
f := o.fs.Load()
if f != nil {
@@ -198,7 +189,6 @@ func (o *srv) Conn(conn net.Conn) {
tr = o.timeoutRead()
tw = o.timeoutWrite()
br = o.buffRead()
bw = o.buffWrite()
)
defer o.fctInfo(lc, rm, libsck.ConnectionClose)
@@ -220,14 +210,12 @@ func (o *srv) Conn(conn net.Conn) {
}
o.fctInfo(lc, rm, libsck.ConnectionRead)
if _, e := io.Copy(br, conn); e != nil {
o.fctError(e)
return
}
o.fctInfo(lc, rm, libsck.ConnectionCloseRead)
if e := conn.(*net.UnixConn).CloseRead(); e != nil {
o.fctError(e)
return
@@ -235,14 +223,7 @@ func (o *srv) Conn(conn net.Conn) {
if h := o.handler(); h != nil {
o.fctInfo(lc, rm, libsck.ConnectionHandler)
h(br, bw)
}
if bw.Len() > 0 {
o.fctInfo(lc, rm, libsck.ConnectionWrite)
if _, e := io.Copy(conn, bw); e != nil {
o.fctError(e)
}
h(br, conn)
}
o.fctInfo(lc, rm, libsck.ConnectionCloseWrite)

1
socket/server/unix/model.go
View File

@@ -59,7 +59,6 @@ type srv struct {
tr *atomic.Value // connection read timeout
tw *atomic.Value // connection write timeout
sr *atomic.Int32 // read buffer size
sw *atomic.Int32 // write buffer size
fs *atomic.Value // file unix socket
fp *atomic.Int64 // file unix perm
}

4
static/pathfile.go
View File

@@ -37,7 +37,7 @@ import (
liberr "github.com/nabbar/golib/errors"
libfpg "github.com/nabbar/golib/file/progress"
libiot "github.com/nabbar/golib/ioutils"
libbuf "github.com/nabbar/golib/ioutils/bufferReadCloser"
)
func (s *staticHandler) _getSize() int64 {
@@ -150,7 +150,7 @@ func (s *staticHandler) _fileBuff(pathFile string) (io.ReadCloser, liberr.Error)
} else if err != nil {
return nil, ErrorFileOpen.Error(err)
} else {
return libiot.NewBufferReadCloser(bytes.NewBuffer(obj)), nil
return libbuf.New(bytes.NewBuffer(obj)), nil
}
}

2
test/test-nutsdb/main.go
View File

@@ -53,7 +53,7 @@ import (
libsh "github.com/nabbar/golib/shell"
libvrs "github.com/nabbar/golib/version"
"github.com/nutsdb/nutsdb"
"github.com/vbauerster/mpb/v5"
"github.com/vbauerster/mpb/v8"
)
const (

136
test/test-progressbar-complex-mpb/main.go Normal file
View File

@@ -0,0 +1,136 @@
//go:build examples
// +build examples
/*
* MIT License
*
* Copyright (c) 2020 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
package main
import (
"fmt"
"math/rand"
"sync"
"time"
"github.com/vbauerster/mpb/v8"
"github.com/vbauerster/mpb/v8/decor"
)
func main() {
var (
tot = int64(1000)
inc = int64(1)
nbb = 5
bar = make([]*mpb.Bar, 0)
wgp = sync.WaitGroup{}
msg = "done"
pct = []decor.Decorator{
decor.Percentage(decor.WC{W: 5, C: 0}),
decor.Name(" | "),
decor.EwmaSpeed(decor.SizeB1024(0), "% .2f", 60),
}
cnt = decor.Counters(decor.SizeB1024(0), "% .2f / % .2f", decor.WC{W: 20, C: decor.DextraSpace})
)
pb := mpb.New(
mpb.WithWidth(64),
mpb.WithRefreshRate(200*time.Millisecond),
)
for i := 0; i < nbb; i++ {
name := "Complex Bar"
job := fmt.Sprintf(" | step %d | ", i)
if i > 0 {
pr := []decor.Decorator{
decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}),
decor.Name(job, decor.WC{W: len(job) + 1, C: decor.DidentRight | decor.DextraSpace}),
cnt,
decor.Name(" ", decor.WC{W: 3, C: decor.DidentRight | decor.DextraSpace}),
decor.OnComplete(decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: len(msg) + 1, C: 0}), msg),
}
if i%2 != 1 {
bar = append(bar, pb.AddBar(tot,
mpb.BarQueueAfter(bar[i-1]),
mpb.BarFillerClearOnComplete(),
mpb.PrependDecorators(pr...),
mpb.AppendDecorators(pct...),
))
} else {
bar = append(bar, pb.AddBar(0,
mpb.BarQueueAfter(bar[i-1]),
mpb.BarFillerClearOnComplete(),
mpb.PrependDecorators(pr...),
mpb.AppendDecorators(pct...),
))
}
} else {
pr := []decor.Decorator{
decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}),
decor.Name(job, decor.WC{W: len(job) + 1, C: decor.DidentRight | decor.DextraSpace}),
cnt,
decor.Name(" ", decor.WC{W: 3, C: decor.DidentRight | decor.DextraSpace}),
}
bar = append(bar, pb.AddBar(0,
mpb.BarFillerClearOnComplete(),
mpb.PrependDecorators(pr...),
mpb.AppendDecorators(pct...),
))
}
}
for i := range bar {
wgp.Add(1)
go func(nb int, done func()) {
defer done()
rand.Seed(999)
for {
if nb > 0 && !(bar[nb-1].Aborted() || bar[nb-1].Completed()) {
time.Sleep(time.Second)
} else if bar[nb].Current() == 0 {
bar[nb].SetTotal(tot, false)
bar[nb].SetRefill(0)
}
if bar[nb].Current() < tot {
time.Sleep(time.Duration(rand.Intn(9)+1) * time.Millisecond)
bar[nb].IncrInt64(inc)
} else {
bar[nb].EnableTriggerComplete()
bar[nb].Abort(false)
return
}
}
}(i, wgp.Done)
}
wgp.Wait()
time.Sleep(500 * time.Millisecond)
println("finish complex...")
}

109
test/test-progressbar-complex/main.go
View File

@@ -1,4 +1,3 @@
//+build examples
//go:build examples
// +build examples
@@ -30,16 +29,20 @@
package main
import (
"fmt"
"math/rand"
"time"
"github.com/nabbar/golib/progress"
libsem "github.com/nabbar/golib/semaphore"
"github.com/vbauerster/mpb/v5"
)
func main() {
max := int64(1000000000)
inc := int64(100000)
tot := int64(1000)
inc := int64(1)
nbb := 5
bar := make([]progress.Bar, 0)
println("\n\n\n")
println("Starting complex...")
@@ -51,82 +54,44 @@ func main() {
pb.DeferMain()
}()
brK0 := pb.NewBarKBits("KiB bar", 1, " | init | ", nil)
brK1 := pb.NewBarKBits("KiB bar", max, " | step 1 | ", brK0)
brK2 := pb.NewBarKBits("KiB bar", max, " | step 2 | ", brK1)
brK0.Reset(max, 0)
brK0.Done()
var (
done1 = false
done2 = false
)
for i := int64(0); i < (max / inc); i++ {
time.Sleep(1 * time.Millisecond)
if e := pb.NewWorker(); e != nil {
continue
for i := 0; i < nbb; i++ {
if i > 0 && i%2 == 1 {
bar = append(bar, pb.NewBarKBits("KiB bar", tot, fmt.Sprintf(" | step %d | ", i), bar[i-1]))
} else if i > 0 && i%2 != 1 {
bar = append(bar, pb.NewBarKBits("KiB bar", 0, fmt.Sprintf(" | step %d | ", i), bar[i-1]))
} else {
bar = append(bar, pb.NewBarKBits("KiB bar", 0, fmt.Sprintf(" | step %d | ", i), nil))
}
go func() {
defer func() {
pb.DeferWorker()
}()
//nolint #nosec
/* #nosec */
rand.Seed(999)
for done1 {
if !done1 && brK1.Completed() {
done1 = true
brK0.Done()
brK1.Reset(max, 0)
} else if !done1 {
//nolint #nosec
/* #nosec */
time.Sleep(time.Duration(rand.Intn(9)+1) * time.Millisecond)
}
}
//nolint #nosec
/* #nosec */
time.Sleep(time.Duration((rand.Intn(99)/3)+1) * time.Millisecond)
brK1.Increment64(inc)
}()
}
for i := int64(0); i < (max / inc); i++ {
time.Sleep(1 * time.Millisecond)
for i := range bar {
if e := pb.NewWorker(); e != nil {
continue
}
go func() {
defer func() {
pb.DeferWorker()
}()
panic(e)
} else {
go func(nb int, sem libsem.Sem) {
defer func() {
pb.DeferWorker()
}()
//nolint #nosec
/* #nosec */
rand.Seed(999)
rand.Seed(999)
for done2 {
if !done2 && brK1.Completed() {
done2 = true
brK1.Done()
brK2.Reset(max, 0)
} else if !done2 {
//nolint #nosec
/* #nosec */
time.Sleep(time.Duration(rand.Intn(9)+1) * time.Millisecond)
for {
if nb > 0 && !bar[nb-1].Completed() {
time.Sleep(time.Second)
} else if bar[nb].Current() == 0 {
bar[nb].Reset(tot, 0)
}
if bar[nb].Current() < tot {
time.Sleep(time.Duration(rand.Intn(9)+1) * time.Millisecond)
bar[nb].Increment64(inc)
} else {
bar[nb].Done()
return
}
}
}
//nolint #nosec
/* #nosec */
time.Sleep(time.Duration(rand.Intn(99)+1) * time.Millisecond)
brK2.Increment64(inc)
}()
}(i, pb)
}
}
if e := pb.WaitAll(); e != nil {

89
test/test-progressbar/main.go
View File

@@ -37,21 +37,27 @@ import (
)
func main() {
max := int64(1000000)
inc := int64(100)
tot := int64(1000)
inc := int64(1)
println("\n\n\n")
println("Starting simple...")
pb := progress.NewProgressBar(mpb.WithWidth(64), mpb.WithRefreshRate(200*time.Millisecond))
brE := pb.NewBarSimpleETA("ETA bar", max)
brE.Reset(max, 0)
brE := pb.NewBarSimpleETA("ETA bar", 0)
brE.Reset(tot/2, 0)
brE.Increment64(inc - 1)
brE.Reset(tot, 0)
brC := pb.NewBarSimpleCounter("counter bar", max)
brC.Reset(max, 0)
brC := pb.NewBarSimpleCounter("counter bar", 0)
brC.Reset(tot/2, 0)
brC.Increment64(inc - 1)
brC.Reset(tot, 0)
brK := pb.NewBarSimpleKBits("KiB bar", max)
brK.Reset(max, 0)
brK := pb.NewBarSimpleKBits("KiB bar", 0)
brK.Reset(tot/2, 0)
brK.Increment64(inc - 1)
brK.Reset(tot, 0)
defer func() {
brE.DeferMain()
@@ -59,42 +65,59 @@ func main() {
brK.DeferMain()
}()
for i := int64(0); i < (max / inc); i++ {
for i := int64(0); i < (tot / inc); i++ {
time.Sleep(5 * time.Millisecond)
if e := brE.NewWorker(); e != nil {
println("Error : " + e.Error())
continue
} else {
go func() {
defer brE.DeferWorker()
//nolint #nosec
/* #nosec */
rand.Seed(99)
//nolint #nosec
/* #nosec */
time.Sleep(time.Duration(rand.Intn(9)) * time.Millisecond)
brE.Increment64(inc - 1)
}()
}
if e := brC.NewWorker(); e != nil {
println("Error : " + e.Error())
brE.DeferWorker()
continue
} else {
go func() {
defer brC.DeferWorker()
//nolint #nosec
/* #nosec */
rand.Seed(99)
//nolint #nosec
/* #nosec */
time.Sleep(time.Duration(rand.Intn(9)) * time.Millisecond)
brC.Increment64(inc - 1)
}()
}
if e := brK.NewWorker(); e != nil {
println("Error : " + e.Error())
brE.DeferWorker()
brC.DeferWorker()
continue
}
go func() {
defer func() {
brE.DeferWorker()
brC.DeferWorker()
brK.DeferWorker()
} else {
go func() {
defer brK.DeferWorker()
//nolint #nosec
/* #nosec */
rand.Seed(99)
//nolint #nosec
/* #nosec */
time.Sleep(time.Duration(rand.Intn(9)) * time.Millisecond)
brK.Increment64(inc - 1)
}()
//nolint #nosec
/* #nosec */
rand.Seed(99)
//nolint #nosec
/* #nosec */
time.Sleep(time.Duration(rand.Intn(9)) * time.Millisecond)
brE.Increment64(inc - 1)
brC.Increment64(inc - 1)
brK.Increment64(inc - 1)
}()
}
}
if e := brE.WaitAll(); e != nil {

6
test/test-win-max-nofile/main.go
View File

@@ -32,12 +32,12 @@ import (
"fmt"
"os"
"github.com/nabbar/golib/ioutils"
"github.com/nabbar/golib/ioutils/fileDescriptor"
)
func main() {
println("test to print Max STDIO NOFILE capabilities !!")
c, _, e := ioutils.SystemFileDescriptor(0)
c, _, e := fileDescriptor.SystemFileDescriptor(0)
println(fmt.Sprintf("Actual limit is : %v | err : %v", c, e))
if e != nil {
@@ -45,7 +45,7 @@ func main() {
}
println("test to Change Max STDIO NOFILE capabilities !!")
c, _, e = ioutils.SystemFileDescriptor(c + 512)
c, _, e = fileDescriptor.SystemFileDescriptor(c + 512)
println(fmt.Sprintf("New limit is : %v | err : %v", c, e))
if e != nil {