namespace config updates

pkg/client.go

@@ -1,10 +1,8 @@
 package v1
 
 import (
-	"encoding/base64"
 	"errors"
 	sq "github.com/Masterminds/squirrel"
-	"github.com/ghodss/yaml"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"regexp"
 	"strconv"
@@ -18,15 +16,6 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 )
 
-const (
-	ArtifactRepositoryEndpointKey       = "artifactRepositoryS3Endpoint"
-	ArtifactRepositoryBucketKey         = "artifactRepositoryS3Bucket"
-	ArtifactRepositoryRegionKey         = "artifactRepositoryS3Region"
-	ArtifactRepositoryInsecureKey       = "artifactRepositoryS3Insecure"
-	ArtifactRepositoryAccessKeyValueKey = "artifactRepositoryS3AccessKey"
-	ArtifactRepositorySecretKeyValueKey = "artifactRepositoryS3SecretKey"
-)
-
 type Config = rest.Config
 
 type DB = sqlx.DB
@@ -75,64 +64,8 @@ func NewClient(config *Config, db *sqlx.DB) (client *Client, err error) {
 	return &Client{Interface: kubeClient, argoprojV1alpha1: argoClient, DB: db}, nil
 }
 
-func (c *Client) GetSystemConfig() (config map[string]string, err error) {
+func (c *Client) GetS3Client(namespace string, config *ArtifactRepositoryS3Config) (s3Client *s3.Client, err error) {
-	namespace := "onepanel"
+	insecure, err := strconv.ParseBool(config.Insecure)
-	configMap, err := c.GetConfigMap(namespace, "onepanel")
-	if err != nil {
-		return
-	}
-	config = configMap.Data
-
-	secret, err := c.GetSecret(namespace, "onepanel")
-	if err != nil {
-		return
-	}
-	databaseUsername, _ := base64.StdEncoding.DecodeString(secret.Data["databaseUsername"])
-	config["databaseUsername"] = string(databaseUsername)
-	databasePassword, _ := base64.StdEncoding.DecodeString(secret.Data["databasePassword"])
-	config["databasePassword"] = string(databasePassword)
-
-	return
-}
-
-func (c *Client) GetNamespaceConfig(namespace string) (config map[string]string, err error) {
-	configMap, err := c.GetConfigMap(namespace, "onepanel")
-	if err != nil {
-		log.WithFields(log.Fields{
-			"Namespace": namespace,
-			"Error":     err.Error(),
-		}).Error("getNamespaceConfig failed getting config map.")
-		return
-	}
-	config = configMap.Data
-	s3Conf := ArtifactRepositoryS3Config{}
-
-	err = yaml.Unmarshal([]byte(configMap.Data["artifactRepository"]), &s3Conf)
-	config[ArtifactRepositoryEndpointKey] = s3Conf.S3.Endpoint
-	config[ArtifactRepositoryBucketKey] = s3Conf.S3.Bucket
-	config[ArtifactRepositoryRegionKey] = s3Conf.S3.Region
-	config[ArtifactRepositoryInsecureKey] = s3Conf.S3.Insecure
-	config[ArtifactRepositoryAccessKeyValueKey] = s3Conf.S3.AccessKeySecret.Key
-	config[ArtifactRepositorySecretKeyValueKey] = s3Conf.S3.SecretKeySecret.Key
-
-	secret, err := c.GetSecret(namespace, "onepanel")
-	if err != nil {
-		log.WithFields(log.Fields{
-			"Namespace": namespace,
-			"Error":     err.Error(),
-		}).Error("getNamespaceConfig failed getting secret.")
-		return
-	}
-	accessKey, _ := base64.StdEncoding.DecodeString(secret.Data[ArtifactRepositoryAccessKeyValueKey])
-	config[ArtifactRepositoryAccessKeyValueKey] = string(accessKey)
-	secretKey, _ := base64.StdEncoding.DecodeString(secret.Data[ArtifactRepositorySecretKeyValueKey])
-	config[ArtifactRepositorySecretKeyValueKey] = string(secretKey)
-
-	return
-}
-
-func (c *Client) GetS3Client(namespace string, config map[string]string) (s3Client *s3.Client, err error) {
-	insecure, err := strconv.ParseBool(config[ArtifactRepositoryInsecureKey])
 	if err != nil {
 		log.WithFields(log.Fields{
 			"Namespace": namespace,
@@ -142,10 +75,10 @@ func (c *Client) GetS3Client(namespace string, config map[string]string) (s3Clie
 		return
 	}
 	s3Client, err = s3.NewClient(s3.Config{
-		Endpoint:  config[ArtifactRepositoryEndpointKey],
+		Endpoint:  config.Endpoint,
-		Region:    config[ArtifactRepositoryRegionKey],
+		Region:    config.Region,
-		AccessKey: config[ArtifactRepositoryAccessKeyValueKey],
+		AccessKey: config.AccessKey,
-		SecretKey: config[ArtifactRepositorySecretKeyValueKey],
+		SecretKey: config.Secretkey,
 		InSecure:  insecure,
 	})
 	if err != nil {

pkg/config.go

@@ -0,0 +1,82 @@
+package v1
+
+import (
+	"encoding/base64"
+	"github.com/ghodss/yaml"
+	"github.com/onepanelio/core/pkg/util"
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc/codes"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (c *Client) getConfigMap(namespace, name string) (configMap *ConfigMap, err error) {
+	cm, err := c.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{})
+	if err != nil {
+		return
+	}
+	configMap = &ConfigMap{
+		Name: name,
+		Data: cm.Data,
+	}
+
+	return
+}
+
+func (c *Client) GetSystemConfig() (config map[string]string, err error) {
+	namespace := "onepanel"
+	configMap, err := c.getConfigMap(namespace, "onepanel")
+	if err != nil {
+		return
+	}
+	config = configMap.Data
+
+	secret, err := c.GetSecret(namespace, "onepanel")
+	if err != nil {
+		return
+	}
+	databaseUsername, _ := base64.StdEncoding.DecodeString(secret.Data["databaseUsername"])
+	config["databaseUsername"] = string(databaseUsername)
+	databasePassword, _ := base64.StdEncoding.DecodeString(secret.Data["databasePassword"])
+	config["databasePassword"] = string(databasePassword)
+
+	return
+}
+
+func (c *Client) GetNamespaceConfig(namespace string) (config *NamespaceConfig, err error) {
+	configMap, err := c.getConfigMap(namespace, "onepanel")
+	if err != nil {
+		log.WithFields(log.Fields{
+			"Namespace": namespace,
+			"Error":     err.Error(),
+		}).Error("getNamespaceConfig failed getting config map.")
+		return
+	}
+	config = &NamespaceConfig{
+		ArtifactRepository: ArtifactRepositoryConfig{},
+	}
+
+	err = yaml.Unmarshal([]byte(configMap.Data["config"]), &config)
+	if err != nil || config.ArtifactRepository.S3 == nil {
+		return nil, util.NewUserError(codes.NotFound, "Artifact repository config not found.")
+	}
+
+	secret, err := c.GetSecret(namespace, "onepanel")
+	if err != nil {
+		log.WithFields(log.Fields{
+			"Namespace": namespace,
+			"Error":     err.Error(),
+		}).Error("getNamespaceConfig failed getting secret.")
+		return
+	}
+
+	// TODO: replace with switch statement to support additional object storage
+	if config.ArtifactRepository.S3 == nil {
+		return nil, util.NewUserError(codes.NotFound, "Artifact repository config not found.")
+	}
+	accessKey, _ := base64.StdEncoding.DecodeString(secret.Data[config.ArtifactRepository.S3.AccessKeySecret.Key])
+	config.ArtifactRepository.S3.AccessKey = string(accessKey)
+	secretKey, _ := base64.StdEncoding.DecodeString(secret.Data[config.ArtifactRepository.S3.SecretKeySecret.Key])
+	config.ArtifactRepository.S3.Secretkey = string(secretKey)
+
+	return
+}

pkg/config_map.go

@@ -1,33 +0,0 @@
-package v1
-
-import (
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func (c *Client) CreateConfigMap(namespace string, configMap *ConfigMap) (err error) {
-	_, err = c.CoreV1().ConfigMaps(namespace).Create(&corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: configMap.Name,
-		},
-		Data: configMap.Data,
-	})
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-func (c *Client) GetConfigMap(namespace, name string) (configMap *ConfigMap, err error) {
-	cm, err := c.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{})
-	if err != nil {
-		return
-	}
-	configMap = &ConfigMap{
-		Name: name,
-		Data: cm.Data,
-	}
-
-	return
-}

pkg/config_map_test.go

@@ -1,31 +0,0 @@
-package v1
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCreateConfigMap(t *testing.T) {
-	c := NewTestClient()
-
-	err := c.CreateConfigMap("namespace", &ConfigMap{
-		Name: "name",
-	})
-	assert.Nil(t, err)
-}
-
-func TestGetConfigMap(t *testing.T) {
-	c := NewTestClient()
-
-	err := c.CreateConfigMap("namespace", &ConfigMap{
-		Name: "name",
-	})
-	assert.Nil(t, err)
-
-	s, err := c.GetConfigMap("namespace", "name")
-	assert.Nil(t, err)
-
-	assert.NotNil(t, s)
-	assert.Equal(t, s.Name, "name")
-}

pkg/config_types.go

@@ -0,0 +1,27 @@
+package v1
+
+type ArtifactRepositoryS3Config struct {
+	KeyFormat       string
+	Bucket          string
+	Endpoint        string
+	Insecure        string
+	Region          string
+	AccessKeySecret struct {
+		Name string
+		Key  string
+	}
+	SecretKeySecret struct {
+		Name string
+		Key  string
+	}
+	AccessKey string
+	Secretkey string
+}
+
+type ArtifactRepositoryConfig struct {
+	S3 *ArtifactRepositoryS3Config
+}
+
+type NamespaceConfig struct {
+	ArtifactRepository ArtifactRepositoryConfig
+}

pkg/types.go

@@ -490,24 +490,6 @@ type File struct {
 	Directory    bool
 }
 
-type ArtifactRepositoryS3Config struct {
-	S3 struct {
-		Bucket          string
-		Endpoint        string
-		Insecure        string
-		Region          string
-		AccessKeySecret struct {
-			Name string
-			Key  string
-		}
-		SecretKeySecret struct {
-			Name string
-			Key  string
-		}
-		Key string
-	}
-}
-
 // Given a path, returns the parent path, asssuming a '/' delimitor
 // Result does not have a trailing slash.
 // -> a/b/c/d would return a/b/c

pkg/workflow_execution.go

@@ -733,7 +733,7 @@ func (c *Client) GetWorkflowExecutionLogs(namespace, uid, podName, containerName
 	var (
 		stream    io.ReadCloser
 		s3Client  *s3.Client
-		config    map[string]string
+		config    *NamespaceConfig
 		endOffset int
 	)
 
@@ -747,10 +747,10 @@ func (c *Client) GetWorkflowExecutionLogs(namespace, uid, podName, containerName
 				"ContainerName": containerName,
 				"Error":         err.Error(),
 			}).Error("Can't get configuration.")
-			return nil, util.NewUserError(codes.PermissionDenied, "Can't get configuration.")
+			return nil, util.NewUserError(codes.NotFound, "Can't get configuration.")
 		}
 
-		s3Client, err = c.GetS3Client(namespace, config)
+		s3Client, err = c.GetS3Client(namespace, config.ArtifactRepository.S3)
 		if err != nil {
 			log.WithFields(log.Fields{
 				"Namespace":     namespace,
@@ -759,7 +759,7 @@ func (c *Client) GetWorkflowExecutionLogs(namespace, uid, podName, containerName
 				"ContainerName": containerName,
 				"Error":         err.Error(),
 			}).Error("Can't connect to S3 storage.")
-			return nil, util.NewUserError(codes.PermissionDenied, "Can't connect to S3 storage.")
+			return nil, util.NewUserError(codes.NotFound, "Can't connect to S3 storage.")
 		}
 
 		opts := s3.GetObjectOptions{}
@@ -769,7 +769,7 @@ func (c *Client) GetWorkflowExecutionLogs(namespace, uid, podName, containerName
 		}
 		opts.SetRange(0, int64(endOffset))
 
-		stream, err = s3Client.GetObject(config[ArtifactRepositoryBucketKey], "artifacts/"+namespace+"/"+uid+"/"+podName+"/"+containerName+".log", opts)
+		stream, err = s3Client.GetObject(config.ArtifactRepository.S3.Bucket, "artifacts/"+namespace+"/"+uid+"/"+podName+"/"+containerName+".log", opts)
 	} else {
 		stream, err = c.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{
 			Container:  containerName,
@@ -822,7 +822,7 @@ func (c *Client) GetWorkflowExecutionMetrics(namespace, uid, podName string) (me
 	var (
 		stream   io.ReadCloser
 		s3Client *s3.Client
-		config   map[string]string
+		config   *NamespaceConfig
 	)
 
 	config, err = c.GetNamespaceConfig(namespace)
@@ -833,10 +833,10 @@ func (c *Client) GetWorkflowExecutionMetrics(namespace, uid, podName string) (me
 			"PodName":   podName,
 			"Error":     err.Error(),
 		}).Error("Can't get configuration.")
-		return nil, util.NewUserError(codes.PermissionDenied, "Can't get configuration.")
+		return nil, util.NewUserError(codes.NotFound, "Can't get configuration.")
 	}
 
-	s3Client, err = c.GetS3Client(namespace, config)
+	s3Client, err = c.GetS3Client(namespace, config.ArtifactRepository.S3)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"Namespace": namespace,
@@ -844,12 +844,12 @@ func (c *Client) GetWorkflowExecutionMetrics(namespace, uid, podName string) (me
 			"PodName":   podName,
 			"Error":     err.Error(),
 		}).Error("Can't connect to S3 storage.")
-		return nil, util.NewUserError(codes.PermissionDenied, "Can't connect to S3 storage.")
+		return nil, util.NewUserError(codes.NotFound, "Can't connect to S3 storage.")
 	}
 
 	opts := s3.GetObjectOptions{}
 
-	stream, err = s3Client.GetObject(config[ArtifactRepositoryBucketKey], "artifacts/"+namespace+"/"+uid+"/"+podName+"/sys-metrics.json", opts)
+	stream, err = s3Client.GetObject(config.ArtifactRepository.S3.Bucket, "artifacts/"+namespace+"/"+uid+"/"+podName+"/sys-metrics.json", opts)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"Namespace": namespace,
@@ -969,13 +969,13 @@ func (c *Client) GetArtifact(namespace, uid, key string) (data []byte, err error
 		return
 	}
 
-	s3Client, err := c.GetS3Client(namespace, config)
+	s3Client, err := c.GetS3Client(namespace, config.ArtifactRepository.S3)
 	if err != nil {
 		return
 	}
 
 	opts := s3.GetObjectOptions{}
-	stream, err := s3Client.GetObject(config[ArtifactRepositoryBucketKey], key, opts)
+	stream, err := s3Client.GetObject(config.ArtifactRepository.S3.Bucket, key, opts)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"Namespace": namespace,
@@ -1000,7 +1000,7 @@ func (c *Client) ListFiles(namespace, key string) (files []*File, err error) {
 		return
 	}
 
-	s3Client, err := c.GetS3Client(namespace, config)
+	s3Client, err := c.GetS3Client(namespace, config.ArtifactRepository.S3)
 	if err != nil {
 		return
 	}
@@ -1015,7 +1015,7 @@ func (c *Client) ListFiles(namespace, key string) (files []*File, err error) {
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)
-	for objInfo := range s3Client.ListObjectsV2(config[ArtifactRepositoryBucketKey], key, false, doneCh) {
+	for objInfo := range s3Client.ListObjectsV2(config.ArtifactRepository.S3.Bucket, key, false, doneCh) {
 		if objInfo.Key == key {
 			continue
 		}