From cf7d52bb19323f6f18a8d5c95c9af742055fded7 Mon Sep 17 00:00:00 2001 From: rushtehrani Date: Mon, 18 May 2020 12:34:05 -0700 Subject: [PATCH] namespace config updates --- pkg/client.go | 79 +++---------------------------------- pkg/config.go | 82 +++++++++++++++++++++++++++++++++++++++ pkg/config_map.go | 33 ---------------- pkg/config_map_test.go | 31 --------------- pkg/config_types.go | 27 +++++++++++++ pkg/types.go | 18 --------- pkg/workflow_execution.go | 28 ++++++------- 7 files changed, 129 insertions(+), 169 deletions(-) create mode 100644 pkg/config.go delete mode 100644 pkg/config_map.go delete mode 100644 pkg/config_map_test.go create mode 100644 pkg/config_types.go diff --git a/pkg/client.go b/pkg/client.go index f4bbada..49bdd28 100644 --- a/pkg/client.go +++ b/pkg/client.go @@ -1,10 +1,8 @@ package v1 import ( - "encoding/base64" "errors" sq "github.com/Masterminds/squirrel" - "github.com/ghodss/yaml" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "regexp" "strconv" @@ -18,15 +16,6 @@ import ( "k8s.io/client-go/tools/clientcmd" ) -const ( - ArtifactRepositoryEndpointKey = "artifactRepositoryS3Endpoint" - ArtifactRepositoryBucketKey = "artifactRepositoryS3Bucket" - ArtifactRepositoryRegionKey = "artifactRepositoryS3Region" - ArtifactRepositoryInsecureKey = "artifactRepositoryS3Insecure" - ArtifactRepositoryAccessKeyValueKey = "artifactRepositoryS3AccessKey" - ArtifactRepositorySecretKeyValueKey = "artifactRepositoryS3SecretKey" -) - type Config = rest.Config type DB = sqlx.DB @@ -75,64 +64,8 @@ func NewClient(config *Config, db *sqlx.DB) (client *Client, err error) { return &Client{Interface: kubeClient, argoprojV1alpha1: argoClient, DB: db}, nil } -func (c *Client) GetSystemConfig() (config map[string]string, err error) { - namespace := "onepanel" - configMap, err := c.GetConfigMap(namespace, "onepanel") - if err != nil { - return - } - config = configMap.Data - - secret, err := c.GetSecret(namespace, "onepanel") - if err != nil { - return - } - databaseUsername, _ := base64.StdEncoding.DecodeString(secret.Data["databaseUsername"]) - config["databaseUsername"] = string(databaseUsername) - databasePassword, _ := base64.StdEncoding.DecodeString(secret.Data["databasePassword"]) - config["databasePassword"] = string(databasePassword) - - return -} - -func (c *Client) GetNamespaceConfig(namespace string) (config map[string]string, err error) { - configMap, err := c.GetConfigMap(namespace, "onepanel") - if err != nil { - log.WithFields(log.Fields{ - "Namespace": namespace, - "Error": err.Error(), - }).Error("getNamespaceConfig failed getting config map.") - return - } - config = configMap.Data - s3Conf := ArtifactRepositoryS3Config{} - - err = yaml.Unmarshal([]byte(configMap.Data["artifactRepository"]), &s3Conf) - config[ArtifactRepositoryEndpointKey] = s3Conf.S3.Endpoint - config[ArtifactRepositoryBucketKey] = s3Conf.S3.Bucket - config[ArtifactRepositoryRegionKey] = s3Conf.S3.Region - config[ArtifactRepositoryInsecureKey] = s3Conf.S3.Insecure - config[ArtifactRepositoryAccessKeyValueKey] = s3Conf.S3.AccessKeySecret.Key - config[ArtifactRepositorySecretKeyValueKey] = s3Conf.S3.SecretKeySecret.Key - - secret, err := c.GetSecret(namespace, "onepanel") - if err != nil { - log.WithFields(log.Fields{ - "Namespace": namespace, - "Error": err.Error(), - }).Error("getNamespaceConfig failed getting secret.") - return - } - accessKey, _ := base64.StdEncoding.DecodeString(secret.Data[ArtifactRepositoryAccessKeyValueKey]) - config[ArtifactRepositoryAccessKeyValueKey] = string(accessKey) - secretKey, _ := base64.StdEncoding.DecodeString(secret.Data[ArtifactRepositorySecretKeyValueKey]) - config[ArtifactRepositorySecretKeyValueKey] = string(secretKey) - - return -} - -func (c *Client) GetS3Client(namespace string, config map[string]string) (s3Client *s3.Client, err error) { - insecure, err := strconv.ParseBool(config[ArtifactRepositoryInsecureKey]) +func (c *Client) GetS3Client(namespace string, config *ArtifactRepositoryS3Config) (s3Client *s3.Client, err error) { + insecure, err := strconv.ParseBool(config.Insecure) if err != nil { log.WithFields(log.Fields{ "Namespace": namespace, @@ -142,10 +75,10 @@ func (c *Client) GetS3Client(namespace string, config map[string]string) (s3Clie return } s3Client, err = s3.NewClient(s3.Config{ - Endpoint: config[ArtifactRepositoryEndpointKey], - Region: config[ArtifactRepositoryRegionKey], - AccessKey: config[ArtifactRepositoryAccessKeyValueKey], - SecretKey: config[ArtifactRepositorySecretKeyValueKey], + Endpoint: config.Endpoint, + Region: config.Region, + AccessKey: config.AccessKey, + SecretKey: config.Secretkey, InSecure: insecure, }) if err != nil { diff --git a/pkg/config.go b/pkg/config.go new file mode 100644 index 0000000..69e442e --- /dev/null +++ b/pkg/config.go @@ -0,0 +1,82 @@ +package v1 + +import ( + "encoding/base64" + "github.com/ghodss/yaml" + "github.com/onepanelio/core/pkg/util" + log "github.com/sirupsen/logrus" + "google.golang.org/grpc/codes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (c *Client) getConfigMap(namespace, name string) (configMap *ConfigMap, err error) { + cm, err := c.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return + } + configMap = &ConfigMap{ + Name: name, + Data: cm.Data, + } + + return +} + +func (c *Client) GetSystemConfig() (config map[string]string, err error) { + namespace := "onepanel" + configMap, err := c.getConfigMap(namespace, "onepanel") + if err != nil { + return + } + config = configMap.Data + + secret, err := c.GetSecret(namespace, "onepanel") + if err != nil { + return + } + databaseUsername, _ := base64.StdEncoding.DecodeString(secret.Data["databaseUsername"]) + config["databaseUsername"] = string(databaseUsername) + databasePassword, _ := base64.StdEncoding.DecodeString(secret.Data["databasePassword"]) + config["databasePassword"] = string(databasePassword) + + return +} + +func (c *Client) GetNamespaceConfig(namespace string) (config *NamespaceConfig, err error) { + configMap, err := c.getConfigMap(namespace, "onepanel") + if err != nil { + log.WithFields(log.Fields{ + "Namespace": namespace, + "Error": err.Error(), + }).Error("getNamespaceConfig failed getting config map.") + return + } + config = &NamespaceConfig{ + ArtifactRepository: ArtifactRepositoryConfig{}, + } + + err = yaml.Unmarshal([]byte(configMap.Data["config"]), &config) + if err != nil || config.ArtifactRepository.S3 == nil { + return nil, util.NewUserError(codes.NotFound, "Artifact repository config not found.") + } + + secret, err := c.GetSecret(namespace, "onepanel") + if err != nil { + log.WithFields(log.Fields{ + "Namespace": namespace, + "Error": err.Error(), + }).Error("getNamespaceConfig failed getting secret.") + return + } + + // TODO: replace with switch statement to support additional object storage + if config.ArtifactRepository.S3 == nil { + return nil, util.NewUserError(codes.NotFound, "Artifact repository config not found.") + } + accessKey, _ := base64.StdEncoding.DecodeString(secret.Data[config.ArtifactRepository.S3.AccessKeySecret.Key]) + config.ArtifactRepository.S3.AccessKey = string(accessKey) + secretKey, _ := base64.StdEncoding.DecodeString(secret.Data[config.ArtifactRepository.S3.SecretKeySecret.Key]) + config.ArtifactRepository.S3.Secretkey = string(secretKey) + + return +} diff --git a/pkg/config_map.go b/pkg/config_map.go deleted file mode 100644 index 8f71c95..0000000 --- a/pkg/config_map.go +++ /dev/null @@ -1,33 +0,0 @@ -package v1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func (c *Client) CreateConfigMap(namespace string, configMap *ConfigMap) (err error) { - _, err = c.CoreV1().ConfigMaps(namespace).Create(&corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMap.Name, - }, - Data: configMap.Data, - }) - if err != nil { - return - } - - return -} - -func (c *Client) GetConfigMap(namespace, name string) (configMap *ConfigMap, err error) { - cm, err := c.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return - } - configMap = &ConfigMap{ - Name: name, - Data: cm.Data, - } - - return -} diff --git a/pkg/config_map_test.go b/pkg/config_map_test.go deleted file mode 100644 index abdec7b..0000000 --- a/pkg/config_map_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package v1 - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCreateConfigMap(t *testing.T) { - c := NewTestClient() - - err := c.CreateConfigMap("namespace", &ConfigMap{ - Name: "name", - }) - assert.Nil(t, err) -} - -func TestGetConfigMap(t *testing.T) { - c := NewTestClient() - - err := c.CreateConfigMap("namespace", &ConfigMap{ - Name: "name", - }) - assert.Nil(t, err) - - s, err := c.GetConfigMap("namespace", "name") - assert.Nil(t, err) - - assert.NotNil(t, s) - assert.Equal(t, s.Name, "name") -} diff --git a/pkg/config_types.go b/pkg/config_types.go new file mode 100644 index 0000000..4de212d --- /dev/null +++ b/pkg/config_types.go @@ -0,0 +1,27 @@ +package v1 + +type ArtifactRepositoryS3Config struct { + KeyFormat string + Bucket string + Endpoint string + Insecure string + Region string + AccessKeySecret struct { + Name string + Key string + } + SecretKeySecret struct { + Name string + Key string + } + AccessKey string + Secretkey string +} + +type ArtifactRepositoryConfig struct { + S3 *ArtifactRepositoryS3Config +} + +type NamespaceConfig struct { + ArtifactRepository ArtifactRepositoryConfig +} diff --git a/pkg/types.go b/pkg/types.go index 1f127da..4952508 100644 --- a/pkg/types.go +++ b/pkg/types.go @@ -490,24 +490,6 @@ type File struct { Directory bool } -type ArtifactRepositoryS3Config struct { - S3 struct { - Bucket string - Endpoint string - Insecure string - Region string - AccessKeySecret struct { - Name string - Key string - } - SecretKeySecret struct { - Name string - Key string - } - Key string - } -} - // Given a path, returns the parent path, asssuming a '/' delimitor // Result does not have a trailing slash. // -> a/b/c/d would return a/b/c diff --git a/pkg/workflow_execution.go b/pkg/workflow_execution.go index c21fb1c..4cb79b1 100644 --- a/pkg/workflow_execution.go +++ b/pkg/workflow_execution.go @@ -733,7 +733,7 @@ func (c *Client) GetWorkflowExecutionLogs(namespace, uid, podName, containerName var ( stream io.ReadCloser s3Client *s3.Client - config map[string]string + config *NamespaceConfig endOffset int ) @@ -747,10 +747,10 @@ func (c *Client) GetWorkflowExecutionLogs(namespace, uid, podName, containerName "ContainerName": containerName, "Error": err.Error(), }).Error("Can't get configuration.") - return nil, util.NewUserError(codes.PermissionDenied, "Can't get configuration.") + return nil, util.NewUserError(codes.NotFound, "Can't get configuration.") } - s3Client, err = c.GetS3Client(namespace, config) + s3Client, err = c.GetS3Client(namespace, config.ArtifactRepository.S3) if err != nil { log.WithFields(log.Fields{ "Namespace": namespace, @@ -759,7 +759,7 @@ func (c *Client) GetWorkflowExecutionLogs(namespace, uid, podName, containerName "ContainerName": containerName, "Error": err.Error(), }).Error("Can't connect to S3 storage.") - return nil, util.NewUserError(codes.PermissionDenied, "Can't connect to S3 storage.") + return nil, util.NewUserError(codes.NotFound, "Can't connect to S3 storage.") } opts := s3.GetObjectOptions{} @@ -769,7 +769,7 @@ func (c *Client) GetWorkflowExecutionLogs(namespace, uid, podName, containerName } opts.SetRange(0, int64(endOffset)) - stream, err = s3Client.GetObject(config[ArtifactRepositoryBucketKey], "artifacts/"+namespace+"/"+uid+"/"+podName+"/"+containerName+".log", opts) + stream, err = s3Client.GetObject(config.ArtifactRepository.S3.Bucket, "artifacts/"+namespace+"/"+uid+"/"+podName+"/"+containerName+".log", opts) } else { stream, err = c.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{ Container: containerName, @@ -822,7 +822,7 @@ func (c *Client) GetWorkflowExecutionMetrics(namespace, uid, podName string) (me var ( stream io.ReadCloser s3Client *s3.Client - config map[string]string + config *NamespaceConfig ) config, err = c.GetNamespaceConfig(namespace) @@ -833,10 +833,10 @@ func (c *Client) GetWorkflowExecutionMetrics(namespace, uid, podName string) (me "PodName": podName, "Error": err.Error(), }).Error("Can't get configuration.") - return nil, util.NewUserError(codes.PermissionDenied, "Can't get configuration.") + return nil, util.NewUserError(codes.NotFound, "Can't get configuration.") } - s3Client, err = c.GetS3Client(namespace, config) + s3Client, err = c.GetS3Client(namespace, config.ArtifactRepository.S3) if err != nil { log.WithFields(log.Fields{ "Namespace": namespace, @@ -844,12 +844,12 @@ func (c *Client) GetWorkflowExecutionMetrics(namespace, uid, podName string) (me "PodName": podName, "Error": err.Error(), }).Error("Can't connect to S3 storage.") - return nil, util.NewUserError(codes.PermissionDenied, "Can't connect to S3 storage.") + return nil, util.NewUserError(codes.NotFound, "Can't connect to S3 storage.") } opts := s3.GetObjectOptions{} - stream, err = s3Client.GetObject(config[ArtifactRepositoryBucketKey], "artifacts/"+namespace+"/"+uid+"/"+podName+"/sys-metrics.json", opts) + stream, err = s3Client.GetObject(config.ArtifactRepository.S3.Bucket, "artifacts/"+namespace+"/"+uid+"/"+podName+"/sys-metrics.json", opts) if err != nil { log.WithFields(log.Fields{ "Namespace": namespace, @@ -969,13 +969,13 @@ func (c *Client) GetArtifact(namespace, uid, key string) (data []byte, err error return } - s3Client, err := c.GetS3Client(namespace, config) + s3Client, err := c.GetS3Client(namespace, config.ArtifactRepository.S3) if err != nil { return } opts := s3.GetObjectOptions{} - stream, err := s3Client.GetObject(config[ArtifactRepositoryBucketKey], key, opts) + stream, err := s3Client.GetObject(config.ArtifactRepository.S3.Bucket, key, opts) if err != nil { log.WithFields(log.Fields{ "Namespace": namespace, @@ -1000,7 +1000,7 @@ func (c *Client) ListFiles(namespace, key string) (files []*File, err error) { return } - s3Client, err := c.GetS3Client(namespace, config) + s3Client, err := c.GetS3Client(namespace, config.ArtifactRepository.S3) if err != nil { return } @@ -1015,7 +1015,7 @@ func (c *Client) ListFiles(namespace, key string) (files []*File, err error) { doneCh := make(chan struct{}) defer close(doneCh) - for objInfo := range s3Client.ListObjectsV2(config[ArtifactRepositoryBucketKey], key, false, doneCh) { + for objInfo := range s3Client.ListObjectsV2(config.ArtifactRepository.S3.Bucket, key, false, doneCh) { if objInfo.Key == key { continue }