diff --git a/artifact/s3aws/error.go b/artifact/s3aws/error.go index 3105549..47c9a31 100644 --- a/artifact/s3aws/error.go +++ b/artifact/s3aws/error.go @@ -41,6 +41,7 @@ const ( ErrorS3AWSNewVers ErrorS3AWSNotFound ErrorS3AWSDownloadError + ErrorS3AWSIOReaderError ) var ( @@ -85,6 +86,8 @@ func getMessage(code errors.CodeError) (message string) { return "requested version is not found" case ErrorS3AWSDownloadError: return "download object requested occurs error" + case ErrorS3AWSIOReaderError: + return "return io reader is empty" } return "" diff --git a/artifact/s3aws/model.go b/artifact/s3aws/model.go index eff6800..a1fa944 100644 --- a/artifact/s3aws/model.go +++ b/artifact/s3aws/model.go @@ -32,6 +32,8 @@ import ( "regexp" "strings" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/hashicorp/go-version" "github.com/nabbar/golib/artifact" "github.com/nabbar/golib/artifact/client" @@ -196,23 +198,15 @@ func (s *s3awsModel) Download(dst ioutils.FileProgress, containName string, rege func (s *s3awsModel) downloadObject(dst ioutils.FileProgress, object string) errors.Error { var ( - r io.ReadCloser - c []io.Closer + r *s3.GetObjectOutput e errors.Error i os.FileInfo j int64 ) defer func() { - if len(c) > 0 { - for _, b := range c { - if b != nil { - _ = b.Close() - } - } - } - if r != nil { - _ = r.Close() + if r != nil && r.Body != nil { + _ = r.Body.Close() } }() @@ -226,11 +220,13 @@ func (s *s3awsModel) downloadObject(dst ioutils.FileProgress, object string) err dst.ResetMax(j) - if r, c, e = s.c.Object().Get(object); e != nil { + if r, e = s.c.Object().Get(object); e != nil { err := ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object)) err.AddParentError(e) return err - } else if _, err := io.Copy(dst, r); err != nil { + } else if r.Body == nil { + return ErrorS3AWSIOReaderError.ErrorParent(getError(errObject, object)) + } else if _, err := io.Copy(dst, r.Body); err != nil { return ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object), err) } else if i, e = dst.FileStat(); e != nil { err := ErrorS3AWSDownloadError.ErrorParent(getError(errObject, object)) diff --git a/aws/aws_suite_test.go b/aws/aws_suite_test.go index 4c0bd6c..07e0d73 100644 --- a/aws/aws_suite_test.go +++ b/aws/aws_suite_test.go @@ -1,11 +1,14 @@ package aws_test import ( + "bytes" "context" + "crypto/rand" "errors" "fmt" "io/ioutil" "net" + "net/http" "net/url" "os" "os/exec" @@ -15,6 +18,8 @@ import ( "testing" "time" + "github.com/nabbar/golib/httpcli" + "github.com/hashicorp/go-uuid" "github.com/nabbar/golib/aws" "github.com/nabbar/golib/aws/configCustom" @@ -46,6 +51,7 @@ var _ = BeforeSuite(func() { var ( err error name string + htp *http.Client ) ctx, cnl = context.WithCancel(context.Background()) @@ -61,7 +67,11 @@ var _ = BeforeSuite(func() { secretKey = password.Generate(64) ) + htp = httpcli.GetClient(uri.Hostname()) + Expect(htp).NotTo(BeNil()) + cfg = configCustom.NewConfig("", accessKey, secretKey, uri, "us-east-1") + Expect(cfg).NotTo(BeNil()) cfg.SetRegion("us-east-1") err = cfg.RegisterRegionAws(nil) @@ -75,10 +85,11 @@ var _ = BeforeSuite(func() { time.Sleep(10 * time.Second) } + time.Sleep(10 * time.Second) println("Minio is waiting on : " + uri.Host) } - cli, err = aws.New(ctx, cfg, nil) + cli, err = aws.New(ctx, cfg, htp) Expect(err).NotTo(HaveOccurred()) Expect(cli).NotTo(BeNil()) @@ -201,3 +212,15 @@ func WaitMinio(host string) bool { return err == nil } + +func randContent(size int) *bytes.Buffer { + p := make([]byte, size) + + _, err := rand.Read(p) + + if err != nil { + panic(err) + } + + return bytes.NewBuffer(p) +} diff --git a/aws/bucket/bucket.go b/aws/bucket/bucket.go index 8a0b04d..0b343ae 100644 --- a/aws/bucket/bucket.go +++ b/aws/bucket/bucket.go @@ -3,146 +3,119 @@ package bucket import ( "fmt" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/nabbar/golib/aws/helper" - "github.com/nabbar/golib/errors" + sdkaws "github.com/aws/aws-sdk-go-v2/aws" + sdksss "github.com/aws/aws-sdk-go-v2/service/s3" + sdkstp "github.com/aws/aws-sdk-go-v2/service/s3/types" + libhlp "github.com/nabbar/golib/aws/helper" + liberr "github.com/nabbar/golib/errors" ) -func (cli *client) Check() errors.Error { - req := cli.s3.HeadBucketRequest(&s3.HeadBucketInput{ +func (cli *client) Check() liberr.Error { + out, err := cli.s3.HeadBucket(cli.GetContext(), &sdksss.HeadBucketInput{ Bucket: cli.GetBucketAws(), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return cli.GetError(err) - } - - if out == nil || out.HeadBucketOutput == nil { + } else if out == nil { //nolint #goerr113 - return helper.ErrorBucketNotFound.ErrorParent(fmt.Errorf("bucket: %s", cli.GetBucketName())) + return libhlp.ErrorBucketNotFound.ErrorParent(fmt.Errorf("bucket: %s", cli.GetBucketName())) } return nil } -func (cli *client) Create() errors.Error { - req := cli.s3.CreateBucketRequest(&s3.CreateBucketInput{ +func (cli *client) Create() liberr.Error { + out, err := cli.s3.CreateBucket(cli.GetContext(), &sdksss.CreateBucketInput{ Bucket: cli.GetBucketAws(), }) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - return cli.GetError(err) -} - -func (cli *client) Delete() errors.Error { - req := cli.s3.DeleteBucketRequest(&s3.DeleteBucketInput{ - Bucket: cli.GetBucketAws(), - }) - - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - return cli.GetError(err) -} - -func (cli *client) List() ([]s3.Bucket, errors.Error) { - req := cli.s3.ListBucketsRequest(nil) - - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { - return make([]s3.Bucket, 0), cli.GetError(err) + return cli.GetError(err) + } else if out == nil || len(*out.Location) == 0 { + return libhlp.ErrorResponse.Error(nil) } - if out == nil || out.Buckets == nil { - return make([]s3.Bucket, 0), helper.ErrorAwsEmpty.Error(nil) + return cli.GetError(err) +} + +func (cli *client) Delete() liberr.Error { + _, err := cli.s3.DeleteBucket(cli.GetContext(), &sdksss.DeleteBucketInput{ + Bucket: cli.GetBucketAws(), + }) + + return cli.GetError(err) +} + +func (cli *client) List() ([]*sdkstp.Bucket, liberr.Error) { + out, err := cli.s3.ListBuckets(cli.GetContext(), nil) + + if err != nil { + return make([]*sdkstp.Bucket, 0), cli.GetError(err) + } else if out == nil || out.Buckets == nil { + return make([]*sdkstp.Bucket, 0), libhlp.ErrorAwsEmpty.Error(nil) } return out.Buckets, nil } -func (cli *client) SetVersioning(state bool) errors.Error { - var status s3.BucketVersioningStatus = helper.STATE_ENABLED +func (cli *client) SetVersioning(state bool) liberr.Error { + var status sdkstp.BucketVersioningStatus = libhlp.STATE_ENABLED if !state { - status = helper.STATE_SUSPENDED + status = libhlp.STATE_SUSPENDED } - vConf := s3.VersioningConfiguration{ - Status: status, - } - input := s3.PutBucketVersioningInput{ - Bucket: cli.GetBucketAws(), - VersioningConfiguration: &vConf, - } - - req := cli.s3.PutBucketVersioningRequest(&input) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) + _, err := cli.s3.PutBucketVersioning(cli.GetContext(), &sdksss.PutBucketVersioningInput{ + Bucket: cli.GetBucketAws(), + VersioningConfiguration: &sdkstp.VersioningConfiguration{ + Status: status, + }, + }) return cli.GetError(err) } -func (cli *client) GetVersioning() (string, errors.Error) { - input := s3.GetBucketVersioningInput{ +func (cli *client) GetVersioning() (string, liberr.Error) { + out, err := cli.s3.GetBucketVersioning(cli.GetContext(), &sdksss.GetBucketVersioningInput{ Bucket: cli.GetBucketAws(), - } - - req := cli.s3.GetBucketVersioningRequest(&input) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - out, err := req.Send(cli.GetContext()) + }) if err != nil { return "", cli.GetError(err) + } else if out == nil { + return "", libhlp.ErrorResponse.Error(nil) } // MarshalValue always return error as nil - v, _ := out.Status.MarshalValue() - - return v, nil + return string(out.Status), nil } -func (cli *client) EnableReplication(srcRoleARN, dstRoleARN, dstBucketName string) errors.Error { - var status s3.ReplicationRuleStatus = helper.STATE_ENABLED +func (cli *client) EnableReplication(srcRoleARN, dstRoleARN, dstBucketName string) liberr.Error { + var status sdkstp.ReplicationRuleStatus = libhlp.STATE_ENABLED - replicationConf := s3.ReplicationConfiguration{ - Role: aws.String(srcRoleARN + "," + dstRoleARN), - Rules: []s3.ReplicationRule{ - { - Destination: &s3.Destination{ - Bucket: aws.String("arn:aws:s3:::" + dstBucketName), + _, err := cli.s3.PutBucketReplication(cli.GetContext(), &sdksss.PutBucketReplicationInput{ + Bucket: cli.GetBucketAws(), + ReplicationConfiguration: &sdkstp.ReplicationConfiguration{ + Role: sdkaws.String(srcRoleARN + "," + dstRoleARN), + Rules: []*sdkstp.ReplicationRule{ + { + Destination: &sdkstp.Destination{ + Bucket: sdkaws.String("arn:aws:s3:::" + dstBucketName), + }, + Status: status, + Prefix: sdkaws.String(""), }, - Status: status, - Prefix: aws.String(""), }, }, - } - - req := cli.s3.PutBucketReplicationRequest(&s3.PutBucketReplicationInput{ - Bucket: cli.GetBucketAws(), - ReplicationConfiguration: &replicationConf, }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - _, err := req.Send(cli.GetContext()) return cli.GetError(err) } -func (cli *client) DeleteReplication() errors.Error { - req := cli.s3.DeleteBucketReplicationRequest(&s3.DeleteBucketReplicationInput{ +func (cli *client) DeleteReplication() liberr.Error { + _, err := cli.s3.DeleteBucketReplication(cli.GetContext(), &sdksss.DeleteBucketReplicationInput{ Bucket: cli.GetBucketAws(), }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - _, err := req.Send(cli.GetContext()) return cli.GetError(err) } diff --git a/aws/bucket/interface.go b/aws/bucket/interface.go index d38d2fb..c1842d1 100644 --- a/aws/bucket/interface.go +++ b/aws/bucket/interface.go @@ -3,37 +3,38 @@ package bucket import ( "context" - "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/nabbar/golib/aws/helper" - "github.com/nabbar/golib/errors" + sdkiam "github.com/aws/aws-sdk-go-v2/service/iam" + sdksss "github.com/aws/aws-sdk-go-v2/service/s3" + sdkstp "github.com/aws/aws-sdk-go-v2/service/s3/types" + libhlp "github.com/nabbar/golib/aws/helper" + ligerr "github.com/nabbar/golib/errors" ) type client struct { - helper.Helper - iam *iam.Client - s3 *s3.Client + libhlp.Helper + iam *sdkiam.Client + s3 *sdksss.Client } type Bucket interface { - Check() errors.Error + Check() ligerr.Error - List() ([]s3.Bucket, errors.Error) - Create() errors.Error - Delete() errors.Error + List() ([]*sdkstp.Bucket, ligerr.Error) + Create() ligerr.Error + Delete() ligerr.Error //FindObject(pattern string) ([]string, errors.Error) - SetVersioning(state bool) errors.Error - GetVersioning() (string, errors.Error) + SetVersioning(state bool) ligerr.Error + GetVersioning() (string, ligerr.Error) - EnableReplication(srcRoleARN, dstRoleARN, dstBucketName string) errors.Error - DeleteReplication() errors.Error + EnableReplication(srcRoleARN, dstRoleARN, dstBucketName string) ligerr.Error + DeleteReplication() ligerr.Error } -func New(ctx context.Context, bucket string, iam *iam.Client, s3 *s3.Client) Bucket { +func New(ctx context.Context, bucket string, iam *sdkiam.Client, s3 *sdksss.Client) Bucket { return &client{ - Helper: helper.New(ctx, bucket), + Helper: libhlp.New(ctx, bucket), iam: iam, s3: s3, } diff --git a/aws/bucket_test.go b/aws/bucket_test.go index cda7273..f6c0c54 100644 --- a/aws/bucket_test.go +++ b/aws/bucket_test.go @@ -1,8 +1,6 @@ package aws_test import ( - "bytes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -27,9 +25,12 @@ var _ = Describe("Bucket", func() { }) Context("With the object", func() { It("Must succeed", func() { - var err error + var ( + err error + rnd = randContent(64 * 1024) + ) - err = cli.Object().MultipartPut("object", bytes.NewReader([]byte("Hello"))) + err = cli.Object().MultipartPut("object", rnd) Expect(err).ToNot(HaveOccurred()) objects, err := cli.Object().Find("object") @@ -50,7 +51,7 @@ var _ = Describe("Bucket", func() { }) /* - * Not Implemented whit minio + * Not Implemented with minio * Context("Versioning", func() { It("Must be possible to enable versioning", func() { diff --git a/aws/configAws/errors.go b/aws/configAws/errors.go index 035185b..5a15c08 100644 --- a/aws/configAws/errors.go +++ b/aws/configAws/errors.go @@ -5,7 +5,7 @@ import ( ) const ( - ErrorAwsError errors.CodeError = iota + errors.MIN_PKG_Aws + 30 + ErrorAwsError errors.CodeError = iota + errors.MIN_PKG_Aws + 40 ErrorConfigLoader ErrorConfigValidator ErrorConfigJsonUnmarshall diff --git a/aws/configAws/interface.go b/aws/configAws/interface.go index e9ca800..9720e9f 100644 --- a/aws/configAws/interface.go +++ b/aws/configAws/interface.go @@ -4,10 +4,10 @@ import ( "encoding/json" "net/http" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/defaults" - "github.com/aws/aws-sdk-go-v2/aws/external" - aws2 "github.com/nabbar/golib/aws" + sdkaws "github.com/aws/aws-sdk-go-v2/aws" + sdkcfg "github.com/aws/aws-sdk-go-v2/config" + sdkcrd "github.com/aws/aws-sdk-go-v2/credentials" + libaws "github.com/nabbar/golib/aws" "github.com/nabbar/golib/errors" ) @@ -15,7 +15,7 @@ func GetConfigModel() interface{} { return configModel{} } -func NewConfigJsonUnmashal(p []byte) (aws2.Config, errors.Error) { +func NewConfigJsonUnmashal(p []byte) (libaws.Config, errors.Error) { c := configModel{} if err := json.Unmarshal(p, &c); err != nil { return nil, ErrorConfigJsonUnmarshall.ErrorParent(err) @@ -23,13 +23,11 @@ func NewConfigJsonUnmashal(p []byte) (aws2.Config, errors.Error) { return &awsModel{ configModel: c, - logLevel: 0, - awsLevel: 0, retryer: nil, }, nil } -func NewConfig(bucket, accessKey, secretKey, region string) aws2.Config { +func NewConfig(bucket, accessKey, secretKey, region string) libaws.Config { return &awsModel{ configModel: configModel{ Region: region, @@ -37,13 +35,11 @@ func NewConfig(bucket, accessKey, secretKey, region string) aws2.Config { SecretKey: secretKey, Bucket: bucket, }, - logLevel: 0, - awsLevel: 0, - retryer: nil, + retryer: nil, } } -func (c *awsModel) Clone() aws2.Config { +func (c *awsModel) Clone() libaws.Config { return &awsModel{ configModel: configModel{ Region: c.Region, @@ -51,36 +47,32 @@ func (c *awsModel) Clone() aws2.Config { SecretKey: c.SecretKey, Bucket: c.Bucket, }, - logLevel: c.logLevel, - awsLevel: c.awsLevel, - retryer: c.retryer, + retryer: c.retryer, } } -func (c *awsModel) GetConfig(cli *http.Client) (aws.Config, errors.Error) { +func (c *awsModel) GetConfig(cli *http.Client) (*sdkaws.Config, errors.Error) { var ( - cfg aws.Config + cfg sdkaws.Config err error ) - if c.AccessKey != "" && c.SecretKey != "" { - cfg = defaults.Config() - cfg.Credentials = aws.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, "") - } else if cfg, err = external.LoadDefaultAWSConfig(); err != nil { - return cfg, ErrorConfigLoader.ErrorParent(err) + if cfg, err = sdkcfg.LoadDefaultConfig(); err != nil { + return nil, ErrorConfigLoader.ErrorParent(err) + } + + if c.AccessKey != "" && c.SecretKey != "" { + cfg.Credentials = sdkcrd.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, "") } - cfg.Logger = &awsLogger{c.logLevel} - cfg.LogLevel = c.awsLevel cfg.Retryer = c.retryer - cfg.EnableEndpointDiscovery = true cfg.Region = c.Region if cli != nil { cfg.HTTPClient = cli } - return cfg, nil + return &cfg, nil } func (c *awsModel) GetBucketName() string { diff --git a/aws/configAws/log.go b/aws/configAws/log.go deleted file mode 100644 index 10718af..0000000 --- a/aws/configAws/log.go +++ /dev/null @@ -1,55 +0,0 @@ -package configAws - -import ( - "reflect" - - "github.com/nabbar/golib/logger" -) - -type awsLogger struct { - logLevel logger.Level -} - -func (l awsLogger) Log(args ...interface{}) { - pattern := "" - - for i := 0; i < len(args); i++ { - //nolint #exhaustive - switch reflect.TypeOf(args[i]).Kind() { - case reflect.String: - pattern += "%s" - default: - pattern += "%v" - } - } - - l.logLevel.Logf("AWS Log : "+pattern, args...) -} - -func LevelPanic() logger.Level { - return logger.PanicLevel -} - -func LevelFatal() logger.Level { - return logger.FatalLevel -} - -func LevelError() logger.Level { - return logger.ErrorLevel -} - -func LevelWarn() logger.Level { - return logger.WarnLevel -} - -func LevelInfo() logger.Level { - return logger.InfoLevel -} - -func LevelDebug() logger.Level { - return logger.DebugLevel -} - -func LevelNoLog() logger.Level { - return logger.NilLevel -} diff --git a/aws/configAws/models.go b/aws/configAws/models.go index fa88833..e3da6d0 100644 --- a/aws/configAws/models.go +++ b/aws/configAws/models.go @@ -6,11 +6,10 @@ import ( "net" "net/url" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/go-playground/validator/v10" + sdkaws "github.com/aws/aws-sdk-go-v2/aws" + libval "github.com/go-playground/validator/v10" "github.com/nabbar/golib/errors" "github.com/nabbar/golib/httpcli" - "github.com/nabbar/golib/logger" ) type configModel struct { @@ -22,23 +21,20 @@ type configModel struct { type awsModel struct { configModel - - logLevel logger.Level - awsLevel aws.LogLevel - retryer aws.Retryer + retryer sdkaws.Retryer } func (c *awsModel) Validate() errors.Error { - val := validator.New() + val := libval.New() err := val.Struct(c) - if e, ok := err.(*validator.InvalidValidationError); ok { + if e, ok := err.(*libval.InvalidValidationError); ok { return ErrorConfigValidator.ErrorParent(e) } out := ErrorConfigValidator.Error(nil) - for _, e := range err.(validator.ValidationErrors) { + for _, e := range err.(libval.ValidationErrors) { //nolint goerr113 out.AddParent(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Field(), e.ActualTag())) } @@ -76,27 +72,23 @@ func (c awsModel) GetEndpoint() *url.URL { return nil } -func (c *awsModel) ResolveEndpoint(service, region string) (aws.Endpoint, error) { - return aws.Endpoint{}, ErrorEndpointInvalid.Error(nil) +func (c *awsModel) ResolveEndpoint(service, region string) (sdkaws.Endpoint, error) { + return sdkaws.Endpoint{}, ErrorEndpointInvalid.Error(nil) } -func (c *awsModel) SetLogLevel(lvl logger.Level) { - c.logLevel = lvl +func (c *awsModel) IsHTTPs() bool { + return true } -func (c *awsModel) SetAWSLogLevel(lvl aws.LogLevel) { - c.awsLevel = lvl -} - -func (c *awsModel) SetRetryer(retryer aws.Retryer) { +func (c *awsModel) SetRetryer(retryer sdkaws.Retryer) { c.retryer = retryer } func (c awsModel) Check(ctx context.Context) errors.Error { var ( - cfg aws.Config + cfg *sdkaws.Config con net.Conn - end aws.Endpoint + end sdkaws.Endpoint adr *url.URL err error e errors.Error diff --git a/aws/configCustom/errors.go b/aws/configCustom/errors.go index da14c98..a227521 100644 --- a/aws/configCustom/errors.go +++ b/aws/configCustom/errors.go @@ -5,7 +5,7 @@ import ( ) const ( - ErrorAwsError errors.CodeError = iota + errors.MIN_PKG_Aws + 30 + ErrorAwsError errors.CodeError = iota + errors.MIN_PKG_Aws + 20 ErrorConfigValidator ErrorConfigJsonUnmarshall ErrorEndpointInvalid diff --git a/aws/configCustom/interface.go b/aws/configCustom/interface.go index 2765183..fa884ea 100644 --- a/aws/configCustom/interface.go +++ b/aws/configCustom/interface.go @@ -6,9 +6,9 @@ import ( "net/url" "strings" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/defaults" - aws2 "github.com/nabbar/golib/aws" + sdkaws "github.com/aws/aws-sdk-go-v2/aws" + sdkcrd "github.com/aws/aws-sdk-go-v2/credentials" + libaws "github.com/nabbar/golib/aws" "github.com/nabbar/golib/errors" ) @@ -16,7 +16,7 @@ func GetConfigModel() interface{} { return Model{} } -func NewConfigJsonUnmashal(p []byte) (aws2.Config, errors.Error) { +func NewConfigJsonUnmashal(p []byte) (libaws.Config, errors.Error) { c := Model{} if err := json.Unmarshal(p, &c); err != nil { return nil, ErrorConfigJsonUnmarshall.ErrorParent(err) @@ -24,14 +24,12 @@ func NewConfigJsonUnmashal(p []byte) (aws2.Config, errors.Error) { return &awsModel{ Model: c, - logLevel: 0, - awsLevel: 0, retryer: nil, mapRegion: nil, }, nil } -func NewConfig(bucket, accessKey, secretKey string, endpoint *url.URL, region string) aws2.Config { +func NewConfig(bucket, accessKey, secretKey string, endpoint *url.URL, region string) libaws.Config { return &awsModel{ Model: Model{ Region: region, @@ -41,14 +39,12 @@ func NewConfig(bucket, accessKey, secretKey string, endpoint *url.URL, region st Bucket: bucket, }, endpoint: endpoint, - logLevel: 0, - awsLevel: 0, retryer: nil, mapRegion: make(map[string]*url.URL), } } -func (c *awsModel) Clone() aws2.Config { +func (c *awsModel) Clone() libaws.Config { m := make(map[string]*url.URL) for r, e := range c.mapRegion { @@ -63,23 +59,19 @@ func (c *awsModel) Clone() aws2.Config { SecretKey: c.SecretKey, Bucket: c.Bucket, }, - logLevel: c.logLevel, - awsLevel: c.awsLevel, retryer: c.retryer, endpoint: c.endpoint, mapRegion: m, } } -func (c *awsModel) GetConfig(cli *http.Client) (aws.Config, errors.Error) { - cfg := defaults.Config() - cfg.Credentials = aws.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, "") - cfg.Logger = &awsLogger{c.logLevel} - cfg.LogLevel = c.awsLevel +func (c *awsModel) GetConfig(cli *http.Client) (*sdkaws.Config, errors.Error) { + + cfg := sdkaws.NewConfig() + + cfg.Credentials = sdkcrd.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, "") cfg.Retryer = c.retryer - cfg.EnableEndpointDiscovery = false - cfg.DisableEndpointHostPrefix = true - cfg.EndpointResolver = aws.EndpointResolverFunc(c.ResolveEndpoint) + cfg.EndpointResolver = sdkaws.EndpointResolverFunc(c.ResolveEndpoint) cfg.Region = c.Region if cli != nil { diff --git a/aws/configCustom/log.go b/aws/configCustom/log.go deleted file mode 100644 index e8dc8fb..0000000 --- a/aws/configCustom/log.go +++ /dev/null @@ -1,55 +0,0 @@ -package configCustom - -import ( - "reflect" - - "github.com/nabbar/golib/logger" -) - -type awsLogger struct { - logLevel logger.Level -} - -func (l awsLogger) Log(args ...interface{}) { - pattern := "" - - for i := 0; i < len(args); i++ { - //nolint #exhaustive - switch reflect.TypeOf(args[i]).Kind() { - case reflect.String: - pattern += "%s" - default: - pattern += "%v" - } - } - - l.logLevel.Logf("AWS Log : "+pattern, args...) -} - -func LevelPanic() logger.Level { - return logger.PanicLevel -} - -func LevelFatal() logger.Level { - return logger.FatalLevel -} - -func LevelError() logger.Level { - return logger.ErrorLevel -} - -func LevelWarn() logger.Level { - return logger.WarnLevel -} - -func LevelInfo() logger.Level { - return logger.InfoLevel -} - -func LevelDebug() logger.Level { - return logger.DebugLevel -} - -func LevelNoLog() logger.Level { - return logger.NilLevel -} diff --git a/aws/configCustom/models.go b/aws/configCustom/models.go index 95e5bd1..ecb70a5 100644 --- a/aws/configCustom/models.go +++ b/aws/configCustom/models.go @@ -7,8 +7,8 @@ import ( "net/url" "strings" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/go-playground/validator/v10" + sdkaws "github.com/aws/aws-sdk-go-v2/aws" + libval "github.com/go-playground/validator/v10" "github.com/nabbar/golib/errors" "github.com/nabbar/golib/httpcli" "github.com/nabbar/golib/logger" @@ -25,25 +25,23 @@ type Model struct { type awsModel struct { Model - logLevel logger.Level - awsLevel aws.LogLevel - retryer aws.Retryer + retryer sdkaws.Retryer endpoint *url.URL mapRegion map[string]*url.URL } func (c *awsModel) Validate() errors.Error { - val := validator.New() + val := libval.New() err := val.Struct(c) if err != nil { - if e, ok := err.(*validator.InvalidValidationError); ok { + if e, ok := err.(*libval.InvalidValidationError); ok { return ErrorConfigValidator.ErrorParent(e) } out := ErrorConfigValidator.Error(nil) - for _, e := range err.(validator.ValidationErrors) { + for _, e := range err.(libval.ValidationErrors) { //nolint goerr113 out.AddParent(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Field(), e.ActualTag())) } @@ -96,7 +94,7 @@ func (c *awsModel) RegisterRegionEndpoint(region string, endpoint *url.URL) erro region = c.Region } - val := validator.New() + val := libval.New() if err := val.Var(endpoint, "url,required"); err != nil { return ErrorEndpointInvalid.ErrorParent(err) @@ -127,7 +125,7 @@ func (c *awsModel) RegisterRegionAws(endpoint *url.URL) errors.Error { return ErrorEndpointInvalid.Error(nil) } - val := validator.New() + val := libval.New() if err := val.Var(endpoint, "url,required"); err != nil { return ErrorEndpointInvalid.ErrorParent(err) } @@ -190,38 +188,34 @@ func (c awsModel) GetEndpoint() *url.URL { return c.endpoint } -func (c *awsModel) ResolveEndpoint(service, region string) (aws.Endpoint, error) { +func (c *awsModel) ResolveEndpoint(service, region string) (sdkaws.Endpoint, error) { if e, ok := c.mapRegion[region]; ok { - return aws.Endpoint{ + return sdkaws.Endpoint{ URL: strings.TrimSuffix(e.String(), "/"), }, nil } if c.Endpoint != "" { - return aws.Endpoint{ + return sdkaws.Endpoint{ URL: strings.TrimSuffix(c.Endpoint, "/"), }, nil } logger.DebugLevel.Logf("Called ResolveEndpoint for service '%s' / region '%s' with nil endpoint", service, region) - return aws.Endpoint{}, ErrorEndpointInvalid.Error(nil) + return sdkaws.Endpoint{}, ErrorEndpointInvalid.Error(nil) } -func (c *awsModel) SetLogLevel(lvl logger.Level) { - c.logLevel = lvl +func (c *awsModel) IsHTTPs() bool { + return c.endpoint.Scheme == "https" } -func (c *awsModel) SetAWSLogLevel(lvl aws.LogLevel) { - c.awsLevel = lvl -} - -func (c *awsModel) SetRetryer(retryer aws.Retryer) { +func (c *awsModel) SetRetryer(retryer sdkaws.Retryer) { c.retryer = retryer } func (c awsModel) Check(ctx context.Context) errors.Error { var ( - cfg aws.Config + cfg *sdkaws.Config con net.Conn err error e errors.Error diff --git a/aws/group/group.go b/aws/group/group.go index 46cd7bf..877f6f2 100644 --- a/aws/group/group.go +++ b/aws/group/group.go @@ -7,10 +7,7 @@ import ( ) func (cli *client) List() (map[string]string, errors.Error) { - req := cli.iam.ListGroupsRequest(&iam.ListGroupsInput{}) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - if out, err := req.Send(cli.GetContext()); err != nil { + if out, err := cli.iam.ListGroups(cli.GetContext(), &iam.ListGroupsInput{}); err != nil { return nil, cli.GetError(err) } else { var res = make(map[string]string) @@ -24,23 +21,17 @@ func (cli *client) List() (map[string]string, errors.Error) { } func (cli *client) Add(groupName string) errors.Error { - req := cli.iam.CreateGroupRequest(&iam.CreateGroupInput{ + _, err := cli.iam.CreateGroup(cli.GetContext(), &iam.CreateGroupInput{ GroupName: aws.String(groupName), }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - _, err := req.Send(cli.GetContext()) return cli.GetError(err) } func (cli *client) Remove(groupName string) errors.Error { - req := cli.iam.DeleteGroupRequest(&iam.DeleteGroupInput{ + _, err := cli.iam.DeleteGroup(cli.GetContext(), &iam.DeleteGroupInput{ GroupName: aws.String(groupName), }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - _, err := req.Send(cli.GetContext()) return cli.GetError(err) } diff --git a/aws/group/policy.go b/aws/group/policy.go index d603bd6..049234b 100644 --- a/aws/group/policy.go +++ b/aws/group/policy.go @@ -7,12 +7,11 @@ import ( ) func (cli *client) PolicyList(groupName string) (map[string]string, errors.Error) { - req := cli.iam.ListAttachedGroupPoliciesRequest(&iam.ListAttachedGroupPoliciesInput{ + out, err := cli.iam.ListAttachedGroupPolicies(cli.GetContext(), &iam.ListAttachedGroupPoliciesInput{ GroupName: aws.String(groupName), }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if out, err := req.Send(cli.GetContext()); err != nil { + if err != nil { return nil, cli.GetError(err) } else { var res = make(map[string]string) @@ -26,25 +25,19 @@ func (cli *client) PolicyList(groupName string) (map[string]string, errors.Error } func (cli *client) PolicyAttach(groupName, polArn string) errors.Error { - req := cli.iam.AttachGroupPolicyRequest(&iam.AttachGroupPolicyInput{ + _, err := cli.iam.AttachGroupPolicy(cli.GetContext(), &iam.AttachGroupPolicyInput{ GroupName: aws.String(groupName), PolicyArn: aws.String(polArn), }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - _, err := req.Send(cli.GetContext()) return cli.GetError(err) } func (cli *client) PolicyDetach(groupName, polArn string) errors.Error { - req := cli.iam.DetachGroupPolicyRequest(&iam.DetachGroupPolicyInput{ + _, err := cli.iam.DetachGroupPolicy(cli.GetContext(), &iam.DetachGroupPolicyInput{ GroupName: aws.String(groupName), PolicyArn: aws.String(polArn), }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - _, err := req.Send(cli.GetContext()) return cli.GetError(err) } diff --git a/aws/group/user.go b/aws/group/user.go index b066592..1d9e002 100644 --- a/aws/group/user.go +++ b/aws/group/user.go @@ -7,12 +7,11 @@ import ( ) func (cli *client) UserCheck(username, groupName string) (errors.Error, bool) { - req := cli.iam.ListGroupsForUserRequest(&iam.ListGroupsForUserInput{ + out, err := cli.iam.ListGroupsForUser(cli.GetContext(), &iam.ListGroupsForUserInput{ UserName: aws.String(username), }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if out, err := req.Send(cli.GetContext()); err != nil { + if err != nil { return cli.GetError(err), false } else { for _, g := range out.Groups { @@ -26,12 +25,11 @@ func (cli *client) UserCheck(username, groupName string) (errors.Error, bool) { } func (cli *client) UserList(username string) ([]string, errors.Error) { - req := cli.iam.ListGroupsForUserRequest(&iam.ListGroupsForUserInput{ + out, err := cli.iam.ListGroupsForUser(cli.GetContext(), &iam.ListGroupsForUserInput{ UserName: aws.String(username), }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if out, err := req.Send(cli.GetContext()); err != nil { + if err != nil { return nil, cli.GetError(err) } else { var res = make([]string, 0) @@ -45,25 +43,19 @@ func (cli *client) UserList(username string) ([]string, errors.Error) { } func (cli *client) UserAdd(username, groupName string) errors.Error { - req := cli.iam.AddUserToGroupRequest(&iam.AddUserToGroupInput{ + _, err := cli.iam.AddUserToGroup(cli.GetContext(), &iam.AddUserToGroupInput{ UserName: aws.String(username), GroupName: aws.String(groupName), }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - _, err := req.Send(cli.GetContext()) return cli.GetError(err) } func (cli *client) UserRemove(username, groupName string) errors.Error { - req := cli.iam.RemoveUserFromGroupRequest(&iam.RemoveUserFromGroupInput{ + _, err := cli.iam.RemoveUserFromGroup(cli.GetContext(), &iam.RemoveUserFromGroupInput{ UserName: aws.String(username), GroupName: aws.String(groupName), }) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - - _, err := req.Send(cli.GetContext()) return cli.GetError(err) } diff --git a/aws/helper/errors.go b/aws/helper/errors.go index 4e5ad39..73f4782 100644 --- a/aws/helper/errors.go +++ b/aws/helper/errors.go @@ -4,7 +4,7 @@ import "github.com/nabbar/golib/errors" const ( // minmal are errors.MIN_AVAILABLE + get a hope free range 1000 + 10 for aws-config errors. - ErrorResponse errors.CodeError = iota + errors.MIN_AVAILABLE + 1000 + 10 + ErrorResponse errors.CodeError = iota + errors.MIN_PKG_Aws + 60 ErrorConfigEmpty ErrorAwsEmpty ErrorAws diff --git a/aws/helper/interface.go b/aws/helper/interface.go index 9f9ca18..5215d4f 100644 --- a/aws/helper/interface.go +++ b/aws/helper/interface.go @@ -2,13 +2,11 @@ package helper import ( "context" - "fmt" "io" "net/http" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/awserr" - "github.com/nabbar/golib/errors" + liberr "github.com/nabbar/golib/errors" ) const ( @@ -28,32 +26,25 @@ func New(ctx context.Context, bucket string) Helper { } } -func (cli Helper) GetError(err error) errors.Error { - if err == nil { - return nil +func (cli Helper) GetError(err ...error) liberr.Error { + var er = ErrorAws.Error(nil) + + for _, e := range err { + if e == nil { + continue + } + if n, ok := e.(liberr.Error); ok { + er.AddParentError(n) + } else { + er.AddParent(e) + } } - if aerr, ok := err.(awserr.Error); ok { - return ErrorAws.Error(errors.NewError(0, fmt.Sprintf("(%s) %s", aerr.Code(), aerr.Message()), nil)) + if er.HasParent() { + return er } - if aerr, ok := err.(errors.Error); ok { - return ErrorAws.Error(aerr) - } - - return ErrorAws.ErrorParent(err) -} - -func (cli Helper) ErrorCode(err error) string { - if aerr, ok := err.(awserr.Error); ok { - return aerr.Code() - } - - if aerr, ok := err.(errors.Error); ok { - return aerr.CodeError("") - } - - return "" + return nil } func (cli *Helper) GetContext() context.Context { diff --git a/aws/helper/partSize.go b/aws/helper/partSize.go index 0d7f854..60e5911 100644 --- a/aws/helper/partSize.go +++ b/aws/helper/partSize.go @@ -1,5 +1,13 @@ package helper +import ( + "errors" + "io" + + sdkaws "github.com/aws/aws-sdk-go-v2/aws" + sdktps "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + type PartSize int64 const ( @@ -45,3 +53,87 @@ func (p PartSize) String() string { return "" } + +type ReaderPartSize interface { + io.Reader + NextPart(eTag *string) + CurrPart() int32 + CompPart() *sdktps.CompletedMultipartUpload + IeOEF() bool +} + +func NewReaderPartSize(rd io.Reader, p PartSize) ReaderPartSize { + return &readerPartSize{ + b: rd, + p: p.Int64(), + i: 0, + j: 0, + e: false, + c: nil, + } +} + +type readerPartSize struct { + // buffer + b io.Reader + // partsize + p int64 + // partNumber + i int64 + // current part counter + j int64 + // Is EOF + e bool + // complete part slice + c *sdktps.CompletedMultipartUpload +} + +func (r *readerPartSize) NextPart(eTag *string) { + if r.c == nil { + r.c = &sdktps.CompletedMultipartUpload{ + Parts: nil, + } + } + + if r.c.Parts == nil { + r.c.Parts = make([]*sdktps.CompletedPart, 0) + } + + r.c.Parts = append(r.c.Parts, &sdktps.CompletedPart{ + ETag: eTag, + PartNumber: sdkaws.Int32(int32(r.i)), + }) + + r.i++ + r.j = 0 +} + +func (r readerPartSize) CurrPart() int32 { + return int32(r.i) +} + +func (r readerPartSize) CompPart() *sdktps.CompletedMultipartUpload { + return r.c +} + +func (r readerPartSize) IeOEF() bool { + return r.e +} + +func (r *readerPartSize) Read(p []byte) (n int, err error) { + if r.e || r.j >= r.p { + return 0, io.EOF + } + + if len(p) > int(r.p-r.j) { + p = make([]byte, int(r.p-r.j)) + } + + n, e := r.b.Read(p) + + if errors.Is(e, io.EOF) { + r.e = true + } + + return n, e +} diff --git a/aws/interface.go b/aws/interface.go index 569764f..a3794e0 100644 --- a/aws/interface.go +++ b/aws/interface.go @@ -5,9 +5,10 @@ import ( "net/http" "net/url" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/aws/aws-sdk-go-v2/service/s3" + sdkaws "github.com/aws/aws-sdk-go-v2/aws" + sdksv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + sdkiam "github.com/aws/aws-sdk-go-v2/service/iam" + sdksss "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/nabbar/golib/aws/bucket" "github.com/nabbar/golib/aws/group" "github.com/nabbar/golib/aws/helper" @@ -16,7 +17,6 @@ import ( "github.com/nabbar/golib/aws/role" "github.com/nabbar/golib/aws/user" "github.com/nabbar/golib/errors" - "github.com/nabbar/golib/logger" ) type Config interface { @@ -31,13 +31,11 @@ type Config interface { SetEndpoint(endpoint *url.URL) GetEndpoint() *url.URL - ResolveEndpoint(service, region string) (aws.Endpoint, error) + IsHTTPs() bool + ResolveEndpoint(service, region string) (sdkaws.Endpoint, error) + SetRetryer(retryer sdkaws.Retryer) - SetLogLevel(lvl logger.Level) - SetAWSLogLevel(lvl aws.LogLevel) - SetRetryer(retryer aws.Retryer) - - GetConfig(cli *http.Client) (aws.Config, errors.Error) + GetConfig(cli *http.Client) (*sdkaws.Config, errors.Error) JSON() ([]byte, error) Clone() Config @@ -53,9 +51,9 @@ type AWS interface { Role() role.Role User() user.User - Clone() AWS + Clone() (AWS, errors.Error) Config() Config - ForcePathStyle(enabled bool) + ForcePathStyle(enabled bool) errors.Error GetBucketName() string SetBucketName(bucket string) @@ -65,8 +63,9 @@ type client struct { p bool x context.Context c Config - i *iam.Client - s *s3.Client + i *sdkiam.Client + s *sdksss.Client + h *http.Client } func New(ctx context.Context, cfg Config, httpClient *http.Client) (AWS, errors.Error) { @@ -74,57 +73,118 @@ func New(ctx context.Context, cfg Config, httpClient *http.Client) (AWS, errors. return nil, helper.ErrorConfigEmpty.Error(nil) } - var ( - c aws.Config - i *iam.Client - s *s3.Client - e errors.Error - ) - - if c, e = cfg.GetConfig(httpClient); e != nil { - return nil, e - } - - i = iam.New(c) - s = s3.New(c) - - if httpClient != nil { - i.HTTPClient = httpClient - s.HTTPClient = httpClient - } - if ctx == nil { ctx = context.Background() } - return &client{ + cli := &client{ p: false, x: ctx, c: cfg, - i: i, - s: s, - }, nil -} - -func (c *client) getCliIAM() *iam.Client { - i := iam.New(c.i.Config) - i.HTTPClient = c.i.HTTPClient - return i -} - -func (c *client) getCliS3() *s3.Client { - s := s3.New(c.s.Config) - s.HTTPClient = c.s.HTTPClient - s.ForcePathStyle = c.p - return s -} - -func (c *client) Clone() AWS { - return &client{ - p: c.p, - x: c.x, - c: c.c.Clone(), - i: c.getCliIAM(), - s: c.getCliS3(), + i: nil, + s: nil, + h: httpClient, } + + if i, e := cli.newClientIAM(httpClient); e != nil { + return nil, e + } else { + cli.i = i + } + + if s, e := cli.newClientS3(httpClient); e != nil { + return nil, e + } else { + cli.s = s + } + + return cli, nil +} + +func (cli *client) newClientIAM(httpClient *http.Client) (*sdkiam.Client, errors.Error) { + var ( + c *sdkaws.Config + i *sdkiam.Client + e errors.Error + ) + + if httpClient == nil { + httpClient = cli.h + } + + if c, e = cli.c.GetConfig(httpClient); e != nil { + return nil, e + } + + i = sdkiam.New(sdkiam.Options{ + APIOptions: c.APIOptions, + Credentials: c.Credentials, + EndpointOptions: sdkiam.ResolverOptions{ + DisableHTTPS: cli.c.IsHTTPs(), + }, + EndpointResolver: sdkiam.WithEndpointResolver(c.EndpointResolver, nil), + HTTPSignerV4: sdksv4.NewSigner(), + Region: c.Region, + Retryer: c.Retryer, + HTTPClient: httpClient, + }) + + return i, nil +} + +func (cli *client) newClientS3(httpClient *http.Client) (*sdksss.Client, errors.Error) { + var ( + c *sdkaws.Config + s *sdksss.Client + e errors.Error + ) + + if httpClient == nil { + httpClient = cli.h + } + + if c, e = cli.c.GetConfig(httpClient); e != nil { + return nil, e + } + + s = sdksss.New(sdksss.Options{ + APIOptions: c.APIOptions, + Credentials: c.Credentials, + EndpointOptions: sdksss.ResolverOptions{ + DisableHTTPS: cli.c.IsHTTPs(), + }, + EndpointResolver: sdksss.WithEndpointResolver(c.EndpointResolver, nil), + HTTPSignerV4: sdksv4.NewSigner(), + Region: c.Region, + Retryer: c.Retryer, + HTTPClient: httpClient, + UsePathStyle: cli.p, + }) + + return s, nil +} + +func (c *client) Clone() (AWS, errors.Error) { + cli := &client{ + p: false, + x: c.x, + c: c.c, + i: nil, + s: nil, + h: c.h, + } + + if i, e := cli.newClientIAM(c.h); e != nil { + return nil, e + } else { + cli.i = i + } + + if s, e := cli.newClientS3(c.h); e != nil { + return nil, e + } else { + cli.s = s + } + + return cli, nil } diff --git a/aws/model.go b/aws/model.go index fbfec9b..c08c339 100644 --- a/aws/model.go +++ b/aws/model.go @@ -7,11 +7,19 @@ import ( "github.com/nabbar/golib/aws/policy" "github.com/nabbar/golib/aws/role" "github.com/nabbar/golib/aws/user" + "github.com/nabbar/golib/errors" ) -func (c *client) ForcePathStyle(enabled bool) { +func (c *client) ForcePathStyle(enabled bool) errors.Error { c.p = enabled - c.s.ForcePathStyle = enabled + + if s, e := c.newClientS3(nil); e != nil { + return e + } else { + c.s = s + } + + return nil } func (c *client) Config() Config { @@ -19,27 +27,27 @@ func (c *client) Config() Config { } func (c *client) Bucket() bucket.Bucket { - return bucket.New(c.x, c.c.GetBucketName(), c.getCliIAM(), c.getCliS3()) + return bucket.New(c.x, c.c.GetBucketName(), c.i, c.s) } func (c *client) Group() group.Group { - return group.New(c.x, c.c.GetBucketName(), c.getCliIAM(), c.getCliS3()) + return group.New(c.x, c.c.GetBucketName(), c.i, c.s) } func (c *client) Object() object.Object { - return object.New(c.x, c.c.GetBucketName(), c.getCliIAM(), c.getCliS3()) + return object.New(c.x, c.c.GetBucketName(), c.i, c.s) } func (c *client) Policy() policy.Policy { - return policy.New(c.x, c.c.GetBucketName(), c.getCliIAM(), c.getCliS3()) + return policy.New(c.x, c.c.GetBucketName(), c.i, c.s) } func (c *client) Role() role.Role { - return role.New(c.x, c.c.GetBucketName(), c.getCliIAM(), c.getCliS3()) + return role.New(c.x, c.c.GetBucketName(), c.i, c.s) } func (c *client) User() user.User { - return user.New(c.x, c.c.GetBucketName(), c.getCliIAM(), c.getCliS3()) + return user.New(c.x, c.c.GetBucketName(), c.i, c.s) } func (c *client) GetBucketName() string { diff --git a/aws/object/interface.go b/aws/object/interface.go index 244f54d..436d950 100644 --- a/aws/object/interface.go +++ b/aws/object/interface.go @@ -5,33 +5,34 @@ import ( "context" "io" - "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/aws/aws-sdk-go-v2/service/s3" + sdkiam "github.com/aws/aws-sdk-go-v2/service/iam" + sdksss "github.com/aws/aws-sdk-go-v2/service/s3" + sdktps "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/nabbar/golib/aws/helper" "github.com/nabbar/golib/errors" ) type client struct { helper.Helper - iam *iam.Client - s3 *s3.Client + iam *sdkiam.Client + s3 *sdksss.Client } type Object interface { Find(pattern string) ([]string, errors.Error) Size(object string) (size int64, err errors.Error) - List(continuationToken string) ([]s3.Object, string, int64, errors.Error) - Head(object string) (head map[string]interface{}, meta map[string]string, err errors.Error) - Get(object string) (io.ReadCloser, []io.Closer, errors.Error) + List(continuationToken string) ([]*sdktps.Object, string, int64, errors.Error) + Head(object string) (*sdksss.HeadObjectOutput, errors.Error) + Get(object string) (*sdksss.GetObjectOutput, errors.Error) Put(object string, body *bytes.Reader) errors.Error Delete(object string) errors.Error MultipartPut(object string, body io.Reader) errors.Error - MultipartPutCustom(partSize helper.PartSize, object string, body io.Reader, concurrent int) errors.Error + MultipartPutCustom(partSize helper.PartSize, object string, body io.Reader) errors.Error } -func New(ctx context.Context, bucket string, iam *iam.Client, s3 *s3.Client) Object { +func New(ctx context.Context, bucket string, iam *sdkiam.Client, s3 *sdksss.Client) Object { return &client{ Helper: helper.New(ctx, bucket), iam: iam, diff --git a/aws/object/multipart.go b/aws/object/multipart.go index 5c68789..a151150 100644 --- a/aws/object/multipart.go +++ b/aws/object/multipart.go @@ -2,40 +2,126 @@ package object import ( "io" + "os" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3/s3manager" - "github.com/nabbar/golib/aws/helper" - "github.com/nabbar/golib/errors" + "github.com/nabbar/golib/ioutils" + + sdkaws "github.com/aws/aws-sdk-go-v2/aws" + sdksss "github.com/aws/aws-sdk-go-v2/service/s3" + libhlp "github.com/nabbar/golib/aws/helper" + liberr "github.com/nabbar/golib/errors" ) -const buffSize = 64 * 1024 // double buff of io.copyBuffer +const DefaultPartSize = 5 * libhlp.SizeMegaBytes -func (cli *client) MultipartPut(object string, body io.Reader) errors.Error { - return cli.MultipartPutCustom(helper.SetSizeInt64(s3manager.MinUploadPartSize), object, body, 0) +func (cli *client) MultipartPut(object string, body io.Reader) liberr.Error { + return cli.MultipartPutCustom(DefaultPartSize, object, body) } -func (cli *client) MultipartPutCustom(partSize helper.PartSize, object string, body io.Reader, concurrent int) errors.Error { - uploader := s3manager.NewUploaderWithClient(cli.s3) +func (cli *client) MultipartPutCustom(partSize libhlp.PartSize, object string, body io.Reader) liberr.Error { + var ( + tmp ioutils.FileProgress + rio libhlp.ReaderPartSize + upl *sdksss.CreateMultipartUploadOutput + err error + ) - if partSize > 0 { - uploader.PartSize = partSize.Int64() - } else { - uploader.PartSize = helper.SetSizeInt64(s3manager.MinUploadPartSize).Int64() - } + defer func() { + if tmp != nil { + _ = tmp.Close() + } + }() - if concurrent > 0 { - uploader.Concurrency = concurrent - } - - // Set Buffer size to 64Kb (this is the min size available) - uploader.BufferProvider = s3manager.NewBufferedReadSeekerWriteToPool(buffSize) - - _, err := uploader.UploadWithContext(cli.GetContext(), &s3manager.UploadInput{ - Bucket: cli.GetBucketAws(), - Key: aws.String(object), - Body: body, + upl, err = cli.s3.CreateMultipartUpload(cli.GetContext(), &sdksss.CreateMultipartUploadInput{ + Key: sdkaws.String(object), + Bucket: sdkaws.String(cli.GetBucketName()), }) - return cli.GetError(err) + if err != nil { + return cli.GetError(err) + } else if upl == nil { + return libhlp.ErrorResponse.Error(nil) + } + + rio = libhlp.NewReaderPartSize(body, partSize) + + for !rio.IeOEF() { + var ( + inf os.FileInfo + prt *sdksss.UploadPartOutput + ) + + tmp, err = ioutils.NewFileProgressTemp() + if err != nil { + return cli.multipartCancel(err, upl.UploadId, object) + } + + _, err = io.Copy(tmp, rio) + if err != nil { + return cli.multipartCancel(err, upl.UploadId, object) + } + + _, err = tmp.Seek(0, io.SeekStart) + if err != nil { + return cli.multipartCancel(err, upl.UploadId, object) + } + + inf, err = tmp.FileStat() + if err != nil { + return cli.multipartCancel(err, upl.UploadId, object) + } + + prt, err = cli.s3.UploadPart(cli.GetContext(), &sdksss.UploadPartInput{ + Bucket: sdkaws.String(cli.GetBucketName()), + Body: tmp, + PartNumber: sdkaws.Int32(rio.CurrPart()), + UploadId: upl.UploadId, + Key: sdkaws.String(object), + ContentLength: sdkaws.Int64(inf.Size()), + }) + + _ = tmp.Close() + tmp = nil + + if err != nil { + return cli.multipartCancel(err, upl.UploadId, object) + } else if prt == nil || prt.ETag == nil || len(*prt.ETag) == 0 { + return cli.multipartCancel(libhlp.ErrorResponse.Error(nil), upl.UploadId, object) + } + + rio.NextPart(prt.ETag) + } + + var prt *sdksss.CompleteMultipartUploadOutput + prt, err = cli.s3.CompleteMultipartUpload(cli.GetContext(), &sdksss.CompleteMultipartUploadInput{ + UploadId: upl.UploadId, + MultipartUpload: rio.CompPart(), + Bucket: sdkaws.String(cli.GetBucketName()), + Key: sdkaws.String(object), + }) + + if err != nil { + return cli.multipartCancel(err, upl.UploadId, object) + } else if prt == nil || prt.ETag == nil || len(*prt.ETag) == 0 { + return cli.multipartCancel(libhlp.ErrorResponse.Error(nil), upl.UploadId, object) + } + + return nil +} + +func (cli *client) multipartCancel(err error, updIp *string, object string) liberr.Error { + cnl, e := cli.s3.AbortMultipartUpload(cli.GetContext(), &sdksss.AbortMultipartUploadInput{ + Bucket: sdkaws.String(cli.GetBucketName()), + UploadId: updIp, + Key: sdkaws.String(object), + }) + + if e != nil { + return cli.GetError(e, err) + } else if cnl == nil { + return libhlp.ErrorResponse.Error(cli.GetError(err)) + } else { + return cli.GetError(err) + } + } diff --git a/aws/object/object.go b/aws/object/object.go index fdf83ca..cddb81b 100644 --- a/aws/object/object.go +++ b/aws/object/object.go @@ -2,121 +2,88 @@ package object import ( "bytes" - "io" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" + sdkaws "github.com/aws/aws-sdk-go-v2/aws" + sdksss "github.com/aws/aws-sdk-go-v2/service/s3" + sdktps "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/nabbar/golib/aws/helper" "github.com/nabbar/golib/errors" ) -func (cli *client) List(continuationToken string) ([]s3.Object, string, int64, errors.Error) { - in := s3.ListObjectsV2Input{ +func (cli *client) List(continuationToken string) ([]*sdktps.Object, string, int64, errors.Error) { + in := sdksss.ListObjectsV2Input{ Bucket: cli.GetBucketAws(), } if continuationToken != "" { - in.ContinuationToken = aws.String(continuationToken) + in.ContinuationToken = sdkaws.String(continuationToken) } - req := cli.s3.ListObjectsV2Request(&in) - - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) + out, err := cli.s3.ListObjectsV2(cli.GetContext(), &in) if err != nil { return nil, "", 0, cli.GetError(err) } else if *out.IsTruncated { - return out.Contents, *out.NextContinuationToken, *out.KeyCount, nil + return out.Contents, *out.NextContinuationToken, int64(*out.KeyCount), nil } else { - return out.Contents, "", *out.KeyCount, nil + return out.Contents, "", int64(*out.KeyCount), nil } } -func (cli *client) Get(object string) (io.ReadCloser, []io.Closer, errors.Error) { - req := cli.s3.GetObjectRequest(&s3.GetObjectInput{ +func (cli *client) Get(object string) (*sdksss.GetObjectOutput, errors.Error) { + out, err := cli.s3.GetObject(cli.GetContext(), &sdksss.GetObjectInput{ Bucket: cli.GetBucketAws(), - Key: aws.String(object), + Key: sdkaws.String(object), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(nil, nil) - if err != nil { - cli.Close(req.HTTPRequest, req.HTTPResponse) - return nil, nil, cli.GetError(err) + defer func() { + if out != nil && out.Body != nil { + _ = out.Body.Close() + } + }() + return nil, cli.GetError(err) } else if out.Body == nil { - cli.Close(req.HTTPRequest, req.HTTPResponse) - return nil, nil, helper.ErrorResponse.Error(nil) + return nil, helper.ErrorResponse.Error(nil) } else { - return out.Body, cli.GetCloser(req.HTTPRequest, req.HTTPResponse), nil + return out, nil } } -func (cli *client) Head(object string) (head map[string]interface{}, meta map[string]string, err errors.Error) { - req := cli.s3.HeadObjectRequest(&s3.HeadObjectInput{ +func (cli *client) Head(object string) (*sdksss.HeadObjectOutput, errors.Error) { + out, e := cli.s3.HeadObject(cli.GetContext(), &sdksss.HeadObjectInput{ Bucket: cli.GetBucketAws(), - Key: aws.String(object), + Key: sdkaws.String(object), }) - out, e := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if e != nil { - return nil, nil, cli.GetError(e) - } else if out.Metadata == nil { - return nil, nil, helper.ErrorResponse.Error(nil) + return nil, cli.GetError(e) + } else if out.ETag == nil { + return nil, helper.ErrorResponse.Error(nil) } else { - res := make(map[string]interface{}) - if out.ContentType != nil { - res["ContentType"] = *out.ContentType - } - if out.ContentDisposition != nil { - res["ContentDisposition"] = *out.ContentDisposition - } - if out.ContentEncoding != nil { - res["ContentEncoding"] = *out.ContentEncoding - } - if out.ContentLanguage != nil { - res["ContentLanguage"] = *out.ContentLanguage - } - if out.ContentLength != nil { - res["ContentLength"] = *out.ContentLength - } - - return res, out.Metadata, nil + return out, nil } } func (cli *client) Size(object string) (size int64, err errors.Error) { var ( - h map[string]interface{} - i interface{} - j int64 - o bool + h *sdksss.HeadObjectOutput ) - if h, _, err = cli.Head(object); err != nil { + if h, err = cli.Head(object); err != nil { return - } else if i, o = h["ContentLength"]; !o { - return 0, nil - } else if j, o = i.(int64); !o { - return 0, nil } else { - return j, nil + return *h.ContentLength, nil } } func (cli *client) Put(object string, body *bytes.Reader) errors.Error { - req := cli.s3.PutObjectRequest(&s3.PutObjectInput{ + out, err := cli.s3.PutObject(cli.GetContext(), &sdksss.PutObjectInput{ Bucket: cli.GetBucketAws(), - Key: aws.String(object), + Key: sdkaws.String(object), Body: body, }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return cli.GetError(err) } else if out.ETag == nil { @@ -127,17 +94,14 @@ func (cli *client) Put(object string, body *bytes.Reader) errors.Error { } func (cli *client) Delete(object string) errors.Error { - if _, _, err := cli.Head(object); err != nil { + if _, err := cli.Head(object); err != nil { return err } - req := cli.s3.DeleteObjectRequest(&s3.DeleteObjectInput{ + _, err := cli.s3.DeleteObject(cli.GetContext(), &sdksss.DeleteObjectInput{ Bucket: cli.GetBucketAws(), - Key: aws.String(object), + Key: sdkaws.String(object), }) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - return cli.GetError(err) } diff --git a/aws/object_test.go b/aws/object_test.go index 98b5314..b654f41 100644 --- a/aws/object_test.go +++ b/aws/object_test.go @@ -9,28 +9,26 @@ import ( var _ = Describe("Object", func() { Context("List objects", func() { - It("Must fail with invalid token", func() { + It("Must fail with invalid token -1 ", func() { _, _, _, err := cli.Object().List("token") Expect(err).To(HaveOccurred()) }) }) Context("Put object", func() { - It("Must fail as the bucket doesn't exists", func() { + It("Must fail as the bucket doesn't exists - 2", func() { err := cli.Object().Put("object", bytes.NewReader([]byte(""))) Expect(err).To(HaveOccurred()) }) }) Context("Get object", func() { - It("Must fail as the bucket doesn't exists", func() { - _, c, err := cli.Object().Get("object") + It("Must fail as the bucket doesn't exists - 3", func() { + o, err := cli.Object().Get("object") defer func() { - for _, s := range c { - if s != nil { - _ = s.Close() - } + if o != nil && o.Body != nil { + _ = o.Body.Close() } }() @@ -39,21 +37,21 @@ var _ = Describe("Object", func() { }) Context("Delete object", func() { - It("Must fail as the object doesn't exists", func() { + It("Must fail as the object doesn't exists - 4", func() { err := cli.Object().Delete("object") Expect(err).To(HaveOccurred()) }) }) Context("Multipart Put object", func() { - It("Must fail as the bucket doesn't exists", func() { - err := cli.Object().MultipartPut("object", bytes.NewReader([]byte(""))) + It("Must fail as the bucket doesn't exists - 5", func() { + err := cli.Object().MultipartPut("object", randContent(4*1024)) Expect(err).To(HaveOccurred()) }) }) Context("Delete object", func() { - It("Must fail as the object doesn't exists", func() { + It("Must fail as the object doesn't exists - 6", func() { err := cli.Object().Delete("object") Expect(err).To(HaveOccurred()) }) diff --git a/aws/policy/policies.go b/aws/policy/policies.go index ff74b5b..45285e7 100644 --- a/aws/policy/policies.go +++ b/aws/policy/policies.go @@ -7,10 +7,7 @@ import ( ) func (cli *client) List() (map[string]string, errors.Error) { - req := cli.iam.ListPoliciesRequest(&iam.ListPoliciesInput{}) - - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) + out, err := cli.iam.ListPolicies(cli.GetContext(), &iam.ListPoliciesInput{}) if err != nil { return nil, cli.GetError(err) @@ -26,15 +23,12 @@ func (cli *client) List() (map[string]string, errors.Error) { } func (cli *client) Add(name, desc, policy string) (string, errors.Error) { - req := cli.iam.CreatePolicyRequest(&iam.CreatePolicyInput{ + out, err := cli.iam.CreatePolicy(cli.GetContext(), &iam.CreatePolicyInput{ PolicyName: aws.String(name), Description: aws.String(desc), PolicyDocument: aws.String(policy), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return "", cli.GetError(err) } else { @@ -43,13 +37,10 @@ func (cli *client) Add(name, desc, policy string) (string, errors.Error) { } func (cli *client) Update(polArn, polContents string) errors.Error { - req := cli.iam.ListPolicyVersionsRequest(&iam.ListPolicyVersionsInput{ + out, err := cli.iam.ListPolicyVersions(cli.GetContext(), &iam.ListPolicyVersionsInput{ PolicyArn: aws.String(polArn), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return cli.GetError(err) } else { @@ -59,44 +50,32 @@ func (cli *client) Update(polArn, polContents string) errors.Error { } if !*v.IsDefaultVersion { - reqD := cli.iam.DeletePolicyVersionRequest(&iam.DeletePolicyVersionInput{ + _, _ = cli.iam.DeletePolicyVersion(cli.GetContext(), &iam.DeletePolicyVersionInput{ PolicyArn: aws.String(polArn), VersionId: v.VersionId, }) - - if o, e := reqD.Send(cli.GetContext()); e != nil { - continue - } else if o == nil { - continue - } } } } - reqG := cli.iam.CreatePolicyVersionRequest(&iam.CreatePolicyVersionInput{ + if cli.GetContext().Err() != nil { + return nil + } + + _, err = cli.iam.CreatePolicyVersion(cli.GetContext(), &iam.CreatePolicyVersionInput{ PolicyArn: aws.String(polArn), PolicyDocument: aws.String(polContents), SetAsDefault: aws.Bool(true), }) - if cli.GetContext().Err() != nil { - return nil - } - - _, err = reqG.Send(cli.GetContext()) - defer cli.Close(reqG.HTTPRequest, reqG.HTTPResponse) - return cli.GetError(err) } func (cli *client) Delete(polArn string) errors.Error { - req := cli.iam.ListPolicyVersionsRequest(&iam.ListPolicyVersionsInput{ + out, err := cli.iam.ListPolicyVersions(cli.GetContext(), &iam.ListPolicyVersionsInput{ PolicyArn: aws.String(polArn), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return cli.GetError(err) } else { @@ -106,16 +85,10 @@ func (cli *client) Delete(polArn string) errors.Error { } if !*v.IsDefaultVersion { - reqD := cli.iam.DeletePolicyVersionRequest(&iam.DeletePolicyVersionInput{ + _, _ = cli.iam.DeletePolicyVersion(cli.GetContext(), &iam.DeletePolicyVersionInput{ PolicyArn: aws.String(polArn), VersionId: v.VersionId, }) - - if o, e := reqD.Send(cli.GetContext()); e != nil { - continue - } else if o == nil { - continue - } } } } @@ -124,12 +97,9 @@ func (cli *client) Delete(polArn string) errors.Error { return nil } - reqG := cli.iam.DeletePolicyRequest(&iam.DeletePolicyInput{ + _, err = cli.iam.DeletePolicy(cli.GetContext(), &iam.DeletePolicyInput{ PolicyArn: aws.String(polArn), }) - _, err = reqG.Send(cli.GetContext()) - defer cli.Close(reqG.HTTPRequest, reqG.HTTPResponse) - return cli.GetError(err) } diff --git a/aws/role/interface.go b/aws/role/interface.go index 5297222..a84a1ba 100644 --- a/aws/role/interface.go +++ b/aws/role/interface.go @@ -4,6 +4,7 @@ import ( "context" "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/nabbar/golib/aws/helper" "github.com/nabbar/golib/errors" @@ -16,7 +17,7 @@ type client struct { } type Role interface { - List() ([]iam.Role, errors.Error) + List() ([]*types.Role, errors.Error) Check(name string) (string, errors.Error) Add(name, role string) (string, errors.Error) Delete(roleName string) errors.Error @@ -24,7 +25,7 @@ type Role interface { PolicyAttach(policyARN, roleName string) errors.Error PolicyDetach(policyARN, roleName string) errors.Error - PolicyListAttached(roleName string) ([]iam.AttachedPolicy, errors.Error) + PolicyListAttached(roleName string) ([]*types.AttachedPolicy, errors.Error) } func New(ctx context.Context, bucket string, iam *iam.Client, s3 *s3.Client) Role { diff --git a/aws/role/policy.go b/aws/role/policy.go index 7604e0f..878a94c 100644 --- a/aws/role/policy.go +++ b/aws/role/policy.go @@ -3,17 +3,15 @@ package role import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/nabbar/golib/errors" ) -func (cli *client) PolicyListAttached(roleName string) ([]iam.AttachedPolicy, errors.Error) { - req := cli.iam.ListAttachedRolePoliciesRequest(&iam.ListAttachedRolePoliciesInput{ +func (cli *client) PolicyListAttached(roleName string) ([]*types.AttachedPolicy, errors.Error) { + out, err := cli.iam.ListAttachedRolePolicies(cli.GetContext(), &iam.ListAttachedRolePoliciesInput{ RoleName: aws.String(roleName), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return nil, cli.GetError(err) } else { @@ -22,25 +20,19 @@ func (cli *client) PolicyListAttached(roleName string) ([]iam.AttachedPolicy, er } func (cli *client) PolicyAttach(policyARN, roleName string) errors.Error { - req := cli.iam.AttachRolePolicyRequest(&iam.AttachRolePolicyInput{ + _, err := cli.iam.AttachRolePolicy(cli.GetContext(), &iam.AttachRolePolicyInput{ PolicyArn: aws.String(policyARN), RoleName: aws.String(roleName), }) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - return cli.GetError(err) } func (cli *client) PolicyDetach(policyARN, roleName string) errors.Error { - req := cli.iam.DetachRolePolicyRequest(&iam.DetachRolePolicyInput{ + _, err := cli.iam.DetachRolePolicy(cli.GetContext(), &iam.DetachRolePolicyInput{ PolicyArn: aws.String(policyARN), RoleName: aws.String(roleName), }) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - return cli.GetError(err) } diff --git a/aws/role/role.go b/aws/role/role.go index 138c476..bf85b35 100644 --- a/aws/role/role.go +++ b/aws/role/role.go @@ -3,14 +3,12 @@ package role import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/nabbar/golib/errors" ) -func (cli *client) List() ([]iam.Role, errors.Error) { - req := cli.iam.ListRolesRequest(&iam.ListRolesInput{}) - - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) +func (cli *client) List() ([]*types.Role, errors.Error) { + out, err := cli.iam.ListRoles(cli.GetContext(), &iam.ListRolesInput{}) if err != nil { return nil, cli.GetError(err) @@ -20,13 +18,10 @@ func (cli *client) List() ([]iam.Role, errors.Error) { } func (cli *client) Check(name string) (string, errors.Error) { - req := cli.iam.GetRoleRequest(&iam.GetRoleInput{ + out, err := cli.iam.GetRole(cli.GetContext(), &iam.GetRoleInput{ RoleName: aws.String(name), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return "", cli.GetError(err) } @@ -35,14 +30,11 @@ func (cli *client) Check(name string) (string, errors.Error) { } func (cli *client) Add(name, role string) (string, errors.Error) { - req := cli.iam.CreateRoleRequest(&iam.CreateRoleInput{ + out, err := cli.iam.CreateRole(cli.GetContext(), &iam.CreateRoleInput{ AssumeRolePolicyDocument: aws.String(role), RoleName: aws.String(name), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return "", cli.GetError(err) } else { @@ -51,12 +43,9 @@ func (cli *client) Add(name, role string) (string, errors.Error) { } func (cli *client) Delete(roleName string) errors.Error { - req := cli.iam.DeleteRoleRequest(&iam.DeleteRoleInput{ + _, err := cli.iam.DeleteRole(cli.GetContext(), &iam.DeleteRoleInput{ RoleName: aws.String(roleName), }) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - return cli.GetError(err) } diff --git a/aws/role_test.go b/aws/role_test.go index 81e3555..efddce5 100644 --- a/aws/role_test.go +++ b/aws/role_test.go @@ -2,7 +2,7 @@ package aws_test import ( "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/iam/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -68,11 +68,11 @@ var _ = Describe("Role", func() { Expect(err).To(HaveOccurred()) }) It("Must return 1 policy", func() { - var policies []iam.AttachedPolicy + var policies []*types.AttachedPolicy if minioMode { err = nil - policies = []iam.AttachedPolicy{ + policies = []*types.AttachedPolicy{ { PolicyArn: aws.String(policyArn), PolicyName: aws.String(name), @@ -137,11 +137,11 @@ var _ = Describe("Role", func() { }) Context("List", func() { It("Must return 1 role", func() { - var roles []iam.Role + var roles []*types.Role if minioMode { err = nil - roles = []iam.Role{ + roles = []*types.Role{ { Arn: aws.String(arn), RoleName: aws.String(name), diff --git a/aws/user/access.go b/aws/user/access.go index 6379195..4972358 100644 --- a/aws/user/access.go +++ b/aws/user/access.go @@ -3,23 +3,21 @@ package user import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/nabbar/golib/aws/helper" "github.com/nabbar/golib/errors" ) func (cli *client) AccessList(username string) (map[string]bool, errors.Error) { - var req iam.ListAccessKeysRequest + var req = &iam.ListAccessKeysInput{} if username != "" { - req = cli.iam.ListAccessKeysRequest(&iam.ListAccessKeysInput{ + req = &iam.ListAccessKeysInput{ UserName: aws.String(username), - }) - } else { - req = cli.iam.ListAccessKeysRequest(&iam.ListAccessKeysInput{}) + } } - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) + out, err := cli.iam.ListAccessKeys(cli.GetContext(), req) if err != nil { return nil, cli.GetError(err) @@ -30,9 +28,9 @@ func (cli *client) AccessList(username string) (map[string]bool, errors.Error) { for _, a := range out.AccessKeyMetadata { switch a.Status { - case iam.StatusTypeActive: + case types.StatusTypeActive: res[*a.AccessKeyId] = true - case iam.StatusTypeInactive: + case types.StatusTypeInactive: res[*a.AccessKeyId] = false } } @@ -42,18 +40,15 @@ func (cli *client) AccessList(username string) (map[string]bool, errors.Error) { } func (cli *client) AccessCreate(username string) (string, string, errors.Error) { - var req iam.CreateAccessKeyRequest + var req = &iam.CreateAccessKeyInput{} if username != "" { - req = cli.iam.CreateAccessKeyRequest(&iam.CreateAccessKeyInput{ + req = &iam.CreateAccessKeyInput{ UserName: aws.String(username), - }) - } else { - req = cli.iam.CreateAccessKeyRequest(&iam.CreateAccessKeyInput{}) + } } - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) + out, err := cli.iam.CreateAccessKey(cli.GetContext(), req) if err != nil { return "", "", cli.GetError(err) @@ -65,21 +60,18 @@ func (cli *client) AccessCreate(username string) (string, string, errors.Error) } func (cli *client) AccessDelete(username, accessKey string) errors.Error { - var req iam.DeleteAccessKeyRequest - - if username != "" { - req = cli.iam.DeleteAccessKeyRequest(&iam.DeleteAccessKeyInput{ - AccessKeyId: aws.String(accessKey), - UserName: aws.String(username), - }) - } else { - req = cli.iam.DeleteAccessKeyRequest(&iam.DeleteAccessKeyInput{ - AccessKeyId: aws.String(accessKey), - }) + var req = &iam.DeleteAccessKeyInput{ + AccessKeyId: aws.String(accessKey), } - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) + if username != "" { + req = &iam.DeleteAccessKeyInput{ + AccessKeyId: aws.String(accessKey), + UserName: aws.String(username), + } + } + + _, err := cli.iam.DeleteAccessKey(cli.GetContext(), req) return cli.GetError(err) } diff --git a/aws/user/interface.go b/aws/user/interface.go index ea1870a..b7129a2 100644 --- a/aws/user/interface.go +++ b/aws/user/interface.go @@ -3,39 +3,40 @@ package user import ( "context" - "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/nabbar/golib/aws/helper" - "github.com/nabbar/golib/errors" + sdkiam "github.com/aws/aws-sdk-go-v2/service/iam" + sdkitp "github.com/aws/aws-sdk-go-v2/service/iam/types" + sdksss "github.com/aws/aws-sdk-go-v2/service/s3" + libhlp "github.com/nabbar/golib/aws/helper" + liberr "github.com/nabbar/golib/errors" ) type client struct { - helper.Helper - iam *iam.Client - s3 *s3.Client + libhlp.Helper + iam *sdkiam.Client + s3 *sdksss.Client } type User interface { - List() (map[string]string, errors.Error) - Get(username string) (*iam.User, errors.Error) - Create(username string) errors.Error - Delete(username string) errors.Error + List() (map[string]string, liberr.Error) + Get(username string) (*sdkitp.User, liberr.Error) + Create(username string) liberr.Error + Delete(username string) liberr.Error - PolicyPut(policyDocument, policyName, username string) errors.Error - PolicyAttach(policyARN, username string) errors.Error + PolicyPut(policyDocument, policyName, username string) liberr.Error + PolicyAttach(policyARN, username string) liberr.Error - LoginCheck(username string) errors.Error - LoginCreate(username, password string) errors.Error - LoginDelete(username string) errors.Error + LoginCheck(username string) liberr.Error + LoginCreate(username, password string) liberr.Error + LoginDelete(username string) liberr.Error - AccessList(username string) (map[string]bool, errors.Error) - AccessCreate(username string) (string, string, errors.Error) - AccessDelete(username, accessKey string) errors.Error + AccessList(username string) (map[string]bool, liberr.Error) + AccessCreate(username string) (string, string, liberr.Error) + AccessDelete(username, accessKey string) liberr.Error } -func New(ctx context.Context, bucket string, iam *iam.Client, s3 *s3.Client) User { +func New(ctx context.Context, bucket string, iam *sdkiam.Client, s3 *sdksss.Client) User { return &client{ - Helper: helper.New(ctx, bucket), + Helper: libhlp.New(ctx, bucket), iam: iam, s3: s3, } diff --git a/aws/user/login.go b/aws/user/login.go index 3b319f6..4c73bb2 100644 --- a/aws/user/login.go +++ b/aws/user/login.go @@ -8,26 +8,20 @@ import ( ) func (cli *client) LoginCheck(username string) errors.Error { - req := cli.iam.GetLoginProfileRequest(&iam.GetLoginProfileInput{ + _, err := cli.iam.GetLoginProfile(cli.GetContext(), &iam.GetLoginProfileInput{ UserName: aws.String(username), }) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - return cli.GetError(err) } func (cli *client) LoginCreate(username, password string) errors.Error { - req := cli.iam.CreateLoginProfileRequest(&iam.CreateLoginProfileInput{ + out, err := cli.iam.CreateLoginProfile(cli.GetContext(), &iam.CreateLoginProfileInput{ UserName: aws.String(username), Password: aws.String(password), PasswordResetRequired: aws.Bool(false), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return cli.GetError(err) } else if out.LoginProfile == nil { @@ -38,12 +32,9 @@ func (cli *client) LoginCreate(username, password string) errors.Error { } func (cli *client) LoginDelete(username string) errors.Error { - req := cli.iam.DeleteLoginProfileRequest(&iam.DeleteLoginProfileInput{ + _, err := cli.iam.DeleteLoginProfile(cli.GetContext(), &iam.DeleteLoginProfileInput{ UserName: aws.String(username), }) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - return cli.GetError(err) } diff --git a/aws/user/policy.go b/aws/user/policy.go index e430b33..47e08c6 100644 --- a/aws/user/policy.go +++ b/aws/user/policy.go @@ -7,26 +7,20 @@ import ( ) func (cli *client) PolicyPut(policyDocument, policyName, username string) errors.Error { - req := cli.iam.PutUserPolicyRequest(&iam.PutUserPolicyInput{ + _, err := cli.iam.PutUserPolicy(cli.GetContext(), &iam.PutUserPolicyInput{ PolicyDocument: aws.String(policyDocument), PolicyName: aws.String(policyName), UserName: aws.String(username), }) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - return cli.GetError(err) } func (cli *client) PolicyAttach(policyARN, username string) errors.Error { - req := cli.iam.AttachUserPolicyRequest(&iam.AttachUserPolicyInput{ + _, err := cli.iam.AttachUserPolicy(cli.GetContext(), &iam.AttachUserPolicyInput{ PolicyArn: aws.String(policyARN), UserName: aws.String(username), }) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - return cli.GetError(err) } diff --git a/aws/user/user.go b/aws/user/user.go index 3dbb843..474eab4 100644 --- a/aws/user/user.go +++ b/aws/user/user.go @@ -3,15 +3,13 @@ package user import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/nabbar/golib/aws/helper" "github.com/nabbar/golib/errors" ) func (cli *client) List() (map[string]string, errors.Error) { - req := cli.iam.ListUsersRequest(&iam.ListUsersInput{}) - - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) + out, err := cli.iam.ListUsers(cli.GetContext(), &iam.ListUsersInput{}) if err != nil { return nil, cli.GetError(err) @@ -28,14 +26,11 @@ func (cli *client) List() (map[string]string, errors.Error) { } } -func (cli *client) Get(username string) (*iam.User, errors.Error) { - req := cli.iam.GetUserRequest(&iam.GetUserInput{ +func (cli *client) Get(username string) (*types.User, errors.Error) { + out, err := cli.iam.GetUser(cli.GetContext(), &iam.GetUserInput{ UserName: aws.String(username), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return nil, cli.GetError(err) } @@ -44,13 +39,10 @@ func (cli *client) Get(username string) (*iam.User, errors.Error) { } func (cli *client) Create(username string) errors.Error { - req := cli.iam.CreateUserRequest(&iam.CreateUserInput{ + out, err := cli.iam.CreateUser(cli.GetContext(), &iam.CreateUserInput{ UserName: aws.String(username), }) - out, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - if err != nil { return cli.GetError(err) } else if out.User == nil { @@ -61,12 +53,9 @@ func (cli *client) Create(username string) errors.Error { } func (cli *client) Delete(username string) errors.Error { - req := cli.iam.DeleteUserRequest(&iam.DeleteUserInput{ + _, err := cli.iam.DeleteUser(cli.GetContext(), &iam.DeleteUserInput{ UserName: aws.String(username), }) - _, err := req.Send(cli.GetContext()) - defer cli.Close(req.HTTPRequest, req.HTTPResponse) - return cli.GetError(err) } diff --git a/aws/user_test.go b/aws/user_test.go index b130396..36feb64 100644 --- a/aws/user_test.go +++ b/aws/user_test.go @@ -2,7 +2,7 @@ package aws_test import ( "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/nabbar/golib/password" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -34,11 +34,11 @@ var _ = Describe("User", func() { }) Context("Get", func() { It("Must succeed", func() { - var user *iam.User + var user *types.User if minioMode { err = nil - user = &iam.User{ + user = &types.User{ UserName: aws.String(username), } } else { diff --git a/go.mod b/go.mod index 55c4dc3..85b6b3d 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,17 @@ go 1.15 require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect - github.com/aws/aws-sdk-go-v2 v0.24.0 + github.com/aws/aws-sdk-go-v2 v0.26.0 + github.com/aws/aws-sdk-go-v2/config v0.1.1 + github.com/aws/aws-sdk-go-v2/credentials v0.1.1 + github.com/aws/aws-sdk-go-v2/service/iam v0.26.0 + github.com/aws/aws-sdk-go-v2/service/s3 v0.26.0 github.com/fatih/color v1.9.0 github.com/gin-gonic/gin v1.6.3 github.com/go-ldap/ldap/v3 v3.2.3 github.com/go-ole/go-ole v1.2.4 // indirect github.com/go-playground/validator/v10 v10.4.0 + github.com/go-sql-driver/mysql v1.5.0 // indirect github.com/gobuffalo/envy v1.9.0 // indirect github.com/gobuffalo/packd v1.0.0 // indirect github.com/gobuffalo/packr v1.30.1 @@ -28,7 +33,7 @@ require ( github.com/onsi/ginkgo v1.14.1 github.com/onsi/gomega v1.10.2 github.com/rogpeppe/go-internal v1.6.2 // indirect - github.com/shirou/gopsutil v2.20.8+incompatible + github.com/shirou/gopsutil v2.20.9+incompatible github.com/sirupsen/logrus v1.7.0 github.com/spf13/jwalterweatherman v1.1.0 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect @@ -36,11 +41,11 @@ require ( github.com/ugorji/go v1.1.9 // indirect github.com/vbauerster/mpb/v5 v5.3.0 github.com/xanzy/go-gitlab v0.38.1 - golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a - golang.org/x/net v0.0.0-20200927032502-5d4f70055728 + golang.org/x/crypto v0.0.0-20201001193750-eb9a90e9f9cb + golang.org/x/net v0.0.0-20200930145003-4acb6c075d10 golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 - golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 - golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c // indirect + golang.org/x/sync v0.0.0-20200930132711-30421366ff76 + golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect gopkg.in/yaml.v2 v2.3.0 // indirect )