zishuo/onepanel
1
0
Fork 0
mirror of https://github.com/onepanelio/onepanel.git synced 2025-09-27 10:02:10 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into feat/core.331-support.gcs.artifact.repository

Browse Source
This commit is contained in:
Aleksandr Melnikov
2020-07-14 14:06:47 -07:00
committed by GitHub
parent 6c0d3fe598 a94ae526ca
commit fa96d6ef66
5 changed files with 227 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
db/20200704151301_update_cvat_workspace_template.go → db/go/20200704151301_update_cvat_workspace_template.go
View File

@@ -5,7 +5,6 @@ import (
v1 "github.com/onepanelio/core/pkg"
uid2 "github.com/onepanelio/core/pkg/util/uid"
"github.com/pressly/goose"
"time"
)
const cvatWorkspaceTemplate3 = `# Docker containers that are part of the Workspace
@@ -128,14 +127,20 @@ func init() {
// Up20200704151301 updates the CVAT template to a new version.
func Up20200704151301(tx *sql.Tx) error {
// This code is executed when the migration is applied.
time.Sleep(2 * time.Second)
client, err := getClient()
if err != nil {
return err
}
migrationsRan, err := getRanSQLMigrations(client)
if err != nil {
return err
}
if _, ok := migrationsRan[20200704151301]; ok {
return nil
}
namespaces, err := client.ListOnepanelEnabledNamespaces()
if err != nil {
return err

108
pkg/config.go
View File

@@ -2,121 +2,13 @@ package v1
import (
"encoding/base64"
"fmt"
"github.com/onepanelio/core/pkg/util"
"github.com/onepanelio/core/pkg/util/ptr"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/yaml"
"strings"
)
// SystemConfig is configuration loaded from kubernetes config and secrets that includes information about the
// database, server, etc.
type SystemConfig map[string]string
// NewSystemConfig creates a System config by getting the required data from a ConfigMap and Secret
func NewSystemConfig(configMap *ConfigMap, secret *Secret) (config SystemConfig, err error) {
config = configMap.Data
databaseUsername, err := base64.StdEncoding.DecodeString(secret.Data["databaseUsername"])
if err != nil {
return
}
config["databaseUsername"] = string(databaseUsername)
databasePassword, err := base64.StdEncoding.DecodeString(secret.Data["databasePassword"])
if err != nil {
return
}
config["databasePassword"] = string(databasePassword)
return
}
// GetValue returns the value in the underlying map if it exists, otherwise nil is returned
// If the value does not exist, it is also logged.
func (s SystemConfig) GetValue(name string) *string {
value, ok := s[name]
if !ok {
log.WithFields(log.Fields{
"Method": "SystemConfig.GetValue",
"Name": name,
"Error": "does not exist",
})
return nil
}
return &value
}
// Domain gets the ONEPANEL_DOMAIN value, or nil.
func (s SystemConfig) Domain() *string {
return s.GetValue("ONEPANEL_DOMAIN")
}
// APIURL gets the ONEPANEL_API_URL, or nil.
func (s SystemConfig) APIURL() *string {
return s.GetValue("ONEPANEL_API_URL")
}
// APIProtocol returns either http:// or https:// or nil.
// It is based on the ONEPANEL_API_URL config value and checks if it has https or http
func (s SystemConfig) APIProtocol() *string {
url := s.APIURL()
if url == nil {
return nil
}
if strings.HasPrefix(*url, "https://") {
return ptr.String("https://")
}
return ptr.String("http://")
}
// FQDN gets the ONEPANEL_FQDN value or nil.
func (s SystemConfig) FQDN() *string {
return s.GetValue("ONEPANEL_FQDN")
}
// NodePoolLabel gets the applicationNodePoolLabel from the config or returns nil.
func (s SystemConfig) NodePoolLabel() (label *string) {
return s.GetValue("applicationNodePoolLabel")
}
// NodePoolOptions loads and parses the applicationNodePoolOptions from the config.
// If there is no data, an error is returned.
func (s SystemConfig) NodePoolOptions() (options []*ParameterOption, err error) {
data := s.GetValue("applicationNodePoolOptions")
if data == nil {
return nil, fmt.Errorf("no nodePoolOptions in config")
}
if err = yaml.Unmarshal([]byte(*data), &options); err != nil {
return
}
return
}
// DatabaseDriverName gets the databaseDriverName value, or nil.
func (s SystemConfig) DatabaseDriverName() *string {
return s.GetValue("databaseDriverName")
}
// DatabaseConnection returns system config information to connect to a database
func (s SystemConfig) DatabaseConnection() (driverName, dataSourceName string) {
dataSourceName = fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=disable",
s["databaseHost"], s["databaseUsername"], s["databasePassword"], s["databaseName"])
driverName = *s.DatabaseDriverName()
return
}
func (c *Client) getConfigMap(namespace, name string) (configMap *ConfigMap, err error) {
cm, err := c.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{})
if err != nil {

131
pkg/config_types.go
View File

@@ -2,10 +2,141 @@ package v1
import (
"gopkg.in/yaml.v3"
"encoding/base64"
"fmt"
"github.com/onepanelio/core/pkg/util/ptr"
log "github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/yaml"
"strings"
)
// SystemConfig is configuration loaded from kubernetes config and secrets that includes information about the
// database, server, etc.
type SystemConfig map[string]string
// NodePoolOption extends ParameterOption to support resourceRequirements
type NodePoolOption struct {
ParameterOption
Resources corev1.ResourceRequirements
}
// NewSystemConfig creates a System config by getting the required data from a ConfigMap and Secret
func NewSystemConfig(configMap *ConfigMap, secret *Secret) (config SystemConfig, err error) {
config = configMap.Data
databaseUsername, err := base64.StdEncoding.DecodeString(secret.Data["databaseUsername"])
if err != nil {
return
}
config["databaseUsername"] = string(databaseUsername)
databasePassword, err := base64.StdEncoding.DecodeString(secret.Data["databasePassword"])
if err != nil {
return
}
config["databasePassword"] = string(databasePassword)
return
}
// GetValue returns the value in the underlying map if it exists, otherwise nil is returned
// If the value does not exist, it is also logged.
func (s SystemConfig) GetValue(name string) *string {
value, ok := s[name]
if !ok {
log.WithFields(log.Fields{
"Method": "SystemConfig.GetValue",
"Name": name,
"Error": "does not exist",
})
return nil
}
return &value
}
// Domain gets the ONEPANEL_DOMAIN value, or nil.
func (s SystemConfig) Domain() *string {
return s.GetValue("ONEPANEL_DOMAIN")
}
// APIURL gets the ONEPANEL_API_URL, or nil.
func (s SystemConfig) APIURL() *string {
return s.GetValue("ONEPANEL_API_URL")
}
// APIProtocol returns either http:// or https:// or nil.
// It is based on the ONEPANEL_API_URL config value and checks if it has https or http
func (s SystemConfig) APIProtocol() *string {
url := s.APIURL()
if url == nil {
return nil
}
if strings.HasPrefix(*url, "https://") {
return ptr.String("https://")
}
return ptr.String("http://")
}
// FQDN gets the ONEPANEL_FQDN value or nil.
func (s SystemConfig) FQDN() *string {
return s.GetValue("ONEPANEL_FQDN")
}
// NodePoolLabel gets the applicationNodePoolLabel from the config or returns nil.
func (s SystemConfig) NodePoolLabel() (label *string) {
return s.GetValue("applicationNodePoolLabel")
}
// NodePoolOptions loads and parses the applicationNodePoolOptions from the config.
// If there is no data, an error is returned.
func (s SystemConfig) NodePoolOptions() (options []*NodePoolOption, err error) {
data := s.GetValue("applicationNodePoolOptions")
if data == nil {
return nil, fmt.Errorf("no nodePoolOptions in config")
}
if err = yaml.Unmarshal([]byte(*data), &options); err != nil {
return
}
return
}
// NodePoolOptionByValue returns the nodePoolOption based on a given value
func (s SystemConfig) NodePoolOptionByValue(value string) (option *NodePoolOption, err error) {
options, err := s.NodePoolOptions()
if err != nil {
return
}
for _, opt := range options {
if opt.Value == value {
option = opt
return
}
}
return
}
// DatabaseDriverName gets the databaseDriverName value, or nil.
func (s SystemConfig) DatabaseDriverName() *string {
return s.GetValue("databaseDriverName")
}
// DatabaseConnection returns system config information to connect to a database
func (s SystemConfig) DatabaseConnection() (driverName, dataSourceName string) {
dataSourceName = fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=disable",
s["databaseHost"], s["databaseUsername"], s["databasePassword"], s["databaseName"])
driverName = *s.DatabaseDriverName()
return
}
type ArtifactRepositoryS3Config struct {
KeyFormat string `yaml:"keyFormat"`
Bucket string

85
pkg/workflow_execution.go
View File

@@ -128,6 +128,70 @@ func injectArtifactRepositoryConfig(artifact *wfv1.Artifact, namespaceConfig *Na
}
}
// injectNvidiaGPUFields adds GPU specific fields if there is a GPU request
func injectNvidiaGPUFields(template *wfv1.Template, systemConfig SystemConfig) {
limitsGPUCount := template.Container.Resources.Limits["nvidia.com/gpu"]
requestsGPUCount := template.Container.Resources.Requests["nvidia.com/gpu"]
if limitsGPUCount.IsZero() && requestsGPUCount.IsZero() {
return
}
// TODO: Remove once https://github.com/Azure/AKS/issues/1271 is addressed
if systemConfig["ONEPANEL_PROVIDER"] == "aks" {
template.Volumes = append(template.Volumes, corev1.Volume{
Name: "nvidia",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/usr/local/nvidia",
},
},
})
template.Container.VolumeMounts = append(template.Container.VolumeMounts, corev1.VolumeMount{
Name: "nvidia",
MountPath: "/usr/bin/nvidia-smi",
SubPath: "nvidia-smi",
})
}
}
// injectContainerResourceQuotas adds resource requests and limits if they exist
func injectContainerResourceQuotas(wf *wfv1.Workflow, template *wfv1.Template, systemConfig SystemConfig) {
if template.NodeSelector == nil {
return
}
var value string
for k, v := range template.NodeSelector {
if k == *systemConfig.NodePoolLabel() {
value = v
break
}
}
if value == "" {
return
}
if strings.Contains(value, "{{workflow.") {
parts := strings.Split(strings.Replace(value, "}}", "", -1), ".")
paramName := parts[len(parts)-1]
for _, param := range wf.Spec.Arguments.Parameters {
if param.Name == paramName {
value = *param.Value
break
}
}
}
option, err := systemConfig.NodePoolOptionByValue(value)
if err != nil {
return
}
if option != nil && option.Resources.Limits != nil {
// If a node is selected specifically, match the resources request to limits
option.Resources.Requests = option.Resources.Limits
template.Container.Resources = option.Resources
}
}
func (c *Client) injectAutomatedFields(namespace string, wf *wfv1.Workflow, opts *WorkflowExecutionOptions) (err error) {
if opts.PodGCStrategy == nil {
if wf.Spec.PodGC == nil {
@@ -168,25 +232,27 @@ func (c *Client) injectAutomatedFields(namespace string, wf *wfv1.Workflow, opts
if err != nil {
return err
}
for i, template := range wf.Spec.Templates {
for i := range wf.Spec.Templates {
template := &wf.Spec.Templates[i]
// Do not inject Istio sidecars in workflows
if template.Metadata.Annotations == nil {
wf.Spec.Templates[i].Metadata.Annotations = make(map[string]string)
template.Metadata.Annotations = make(map[string]string)
}
wf.Spec.Templates[i].Metadata.Annotations["sidecar.istio.io/inject"] = "false"
template.Metadata.Annotations["sidecar.istio.io/inject"] = "false"
if template.Container == nil {
continue
}
// Mount dev/shm
wf.Spec.Templates[i].Container.VolumeMounts = append(template.Container.VolumeMounts, corev1.VolumeMount{
template.Container.VolumeMounts = append(template.Container.VolumeMounts, corev1.VolumeMount{
Name: "sys-dshm",
MountPath: "/dev/shm",
})
// Always add output artifacts for metrics but make them optional
wf.Spec.Templates[i].Outputs.Artifacts = append(template.Outputs.Artifacts, wfv1.Artifact{
template.Outputs.Artifacts = append(template.Outputs.Artifacts, wfv1.Artifact{
Name: "sys-metrics",
Path: "/tmp/sys-metrics.json",
Optional: true,
@@ -198,21 +264,24 @@ func (c *Client) injectAutomatedFields(namespace string, wf *wfv1.Workflow, opts
// Extend artifact credentials if only key is provided
for j, artifact := range template.Outputs.Artifacts {
injectArtifactRepositoryConfig(&artifact, namespaceConfig)
wf.Spec.Templates[i].Outputs.Artifacts[j] = artifact
template.Outputs.Artifacts[j] = artifact
}
for j, artifact := range template.Inputs.Artifacts {
injectArtifactRepositoryConfig(&artifact, namespaceConfig)
wf.Spec.Templates[i].Inputs.Artifacts[j] = artifact
template.Inputs.Artifacts[j] = artifact
}
injectContainerResourceQuotas(wf, template, systemConfig)
injectNvidiaGPUFields(template, systemConfig)
//Generate ENV vars from secret, if there is a container present in the workflow
//Get template ENV vars, avoid over-writing them with secret values
env.AddDefaultEnvVarsToContainer(template.Container)
env.PrependEnvVarToContainer(template.Container, "ONEPANEL_API_URL", systemConfig["ONEPANEL_API_URL"])
env.PrependEnvVarToContainer(template.Container, "ONEPANEL_FQDN", systemConfig["ONEPANEL_FQDN"])
env.PrependEnvVarToContainer(template.Container, "ONEPANEL_DOMAIN", systemConfig["ONEPANEL_DOMAIN"])
env.PrependEnvVarToContainer(template.Container, "ONEPANEL_PROVIDER_TYPE", systemConfig["PROVIDER_TYPE"])
env.PrependEnvVarToContainer(template.Container, "ONEPANEL_PROVIDER", systemConfig["ONEPANEL_PROVIDER"])
env.PrependEnvVarToContainer(template.Container, "ONEPANEL_RESOURCE_NAMESPACE", "{{workflow.namespace}}")
env.PrependEnvVarToContainer(template.Container, "ONEPANEL_RESOURCE_UID", "{{workflow.name}}")
}

13
pkg/workspace_template.go
View File

@@ -83,13 +83,20 @@ func generateRuntimeParameters(config SystemConfig) (parameters []Parameter, err
})
// Node pool parameter and options
options, err := config.NodePoolOptions()
nodePoolOptions, err := config.NodePoolOptions()
if err != nil {
return nil, err
}
if len(options) == 0 {
if len(nodePoolOptions) == 0 {
return nil, fmt.Errorf("no node pool options in config")
}
var options []*ParameterOption
for _, option := range nodePoolOptions {
options = append(options, &ParameterOption{
Name: option.Name,
Value: option.Value,
})
}
parameters = append(parameters, Parameter{
Name: "sys-node-pool-label",
@@ -283,7 +290,7 @@ func createStatefulSetManifest(spec *WorkspaceSpec, config map[string]string) (s
env.PrependEnvVarToContainer(container, "ONEPANEL_API_URL", config["ONEPANEL_API_URL"])
env.PrependEnvVarToContainer(container, "ONEPANEL_FQDN", config["ONEPANEL_FQDN"])
env.PrependEnvVarToContainer(container, "ONEPANEL_DOMAIN", config["ONEPANEL_DOMAIN"])
env.PrependEnvVarToContainer(container, "ONEPANEL_PROVIDER_TYPE", config["PROVIDER_TYPE"])
env.PrependEnvVarToContainer(container, "ONEPANEL_PROVIDER", config["ONEPANEL_PROVIDER"])
env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_NAMESPACE", "{{workflow.namespace}}")
env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_UID", "{{workflow.parameters.sys-name}}")