feat: fix issues with comments and auto injection of options for select.nodepool and workflow templates.

pkg/common_types.go

@@ -85,6 +85,11 @@ func ParseParametersFromManifest(manifest []byte) ([]Parameter, error) {
 		if parameter.Visibility == nil {
 			parameter.Visibility = ptr.String("public")
 		}
+
+		if parameter.Type == "select.nodepool" {
+			parameter.Options = make([]*ParameterOption, 0)
+			parameter.Value = ptr.String("default")
+		}
 	}
 
 	if err := IsValidParameters(manifestResult.Arguments.Parameters); err != nil {

pkg/util/extensions/extensions.go

@@ -0,0 +1,185 @@
+package extensions
+
+import (
+	"fmt"
+	"gopkg.in/yaml.v3"
+	"strings"
+)
+
+type NodePair struct {
+	Key   *yaml.Node
+	Value *yaml.Node
+}
+
+type YamlIndex struct {
+	parts []string
+}
+
+// String returns the YamlIndex indicated by the parts separated by "."
+// e.g. parent.children.favoriteNumber
+func (y *YamlIndex) String() string {
+	return strings.Join(y.parts, ".")
+}
+
+// CreateYamlIndex creates a YamlIndex that specifies the Key via string parts.
+// e.g. a key maybe be: parent.child.favoriteNumber and the returned YamlIndex would reflect this.
+func CreateYamlIndex(parts ...string) *YamlIndex {
+	copyParts := make([]string, len(parts))
+
+	for i, part := range parts {
+		copyParts[i] = part
+	}
+
+	return &YamlIndex{
+		parts: copyParts,
+	}
+}
+
+func HasNode(root *yaml.Node, key *YamlIndex) bool {
+	if key == nil || len(key.parts) == 0 {
+		return false
+	}
+
+	currentNode := root
+	if len(root.Content) == 1 {
+		currentNode = root.Content[0]
+	}
+
+	for _, keyPart := range key.parts {
+		found := false
+		for j := 0; j < len(currentNode.Content)-1; j += 2 {
+			keyNode := currentNode.Content[j]
+			valueNode := currentNode.Content[j+1]
+
+			if keyNode.Value == keyPart {
+				currentNode = valueNode
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			return false
+		}
+	}
+
+	return true
+}
+
+// TODO support indexes
+func GetNode(root *yaml.Node, key *YamlIndex) (*yaml.Node, error) {
+	if key == nil || len(key.parts) == 0 {
+		return root, nil
+	}
+
+	currentNode := root
+	if len(root.Content) == 1 {
+		currentNode = root.Content[0]
+	}
+
+	for _, keyPart := range key.parts {
+		found := false
+		for j := 0; j < len(currentNode.Content)-1; j += 2 {
+			keyNode := currentNode.Content[j]
+			valueNode := currentNode.Content[j+1]
+
+			if keyNode.Value == keyPart {
+				currentNode = valueNode
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			return nil, fmt.Errorf("%v not found - stopped at %v", key.String(), keyPart)
+		}
+	}
+
+	return currentNode, nil
+}
+
+func SetKeyValue(node *yaml.Node, key string, value string) error {
+	if node.Kind != yaml.MappingNode {
+		return fmt.Errorf("not a mapping node")
+	}
+
+	for i := 0; i < len(node.Content)-1; i += 2 {
+		keyNode := node.Content[i]
+		valueNode := node.Content[i+1]
+
+		if keyNode.Value == key {
+			valueNode.Value = value
+			break
+		}
+	}
+
+	return nil
+}
+
+// HasKeyValue checks if the node (assumed to be a mapping node) has a key with the given value.
+// If it does not, (false, nil) is returned. If there is an error, like a key not existing, an error is returned.
+func HasKeyValue(node *yaml.Node, key string, value string) (bool, error) {
+	if node.Kind != yaml.MappingNode {
+		return false, fmt.Errorf("not a mapping node")
+	}
+
+	for i := 0; i < len(node.Content)-1; i += 2 {
+		keyNode := node.Content[i]
+		valueNode := node.Content[i+1]
+
+		if keyNode.Value == key {
+			return valueNode.Value == value, nil
+		}
+	}
+
+	return false, nil
+}
+
+func Iterate(root *yaml.Node, callable func(parent, value *yaml.Node)) {
+	for _, child := range root.Content {
+		callable(root, child)
+		Iterate(child, callable)
+	}
+}
+
+func DeleteNode(node *yaml.Node, key *YamlIndex) error {
+	if node.Kind != yaml.MappingNode {
+		return fmt.Errorf("not a mapping node")
+	}
+
+	currentNode := node
+	for i, keyPart := range key.parts {
+		found := false
+		for j := 0; j < len(currentNode.Content)-1; j += 2 {
+			keyNode := currentNode.Content[j]
+			valueNode := currentNode.Content[j+1]
+
+			if keyNode.Value == keyPart {
+				if i != (len(key.parts) - 1) {
+					currentNode = valueNode
+				}
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			return fmt.Errorf("%v not found - stopped at %v", key.String(), keyPart)
+		}
+	}
+
+	keptNodes := make([]*yaml.Node, 0)
+	finalKey := key.parts[len(key.parts)-1]
+	for i := 0; i < len(currentNode.Content)-1; i += 2 {
+		keyNode := node.Content[i]
+		valueNode := node.Content[i+1]
+
+		if keyNode.Value != finalKey {
+			keptNodes = append(keptNodes, keyNode, valueNode)
+		}
+	}
+
+	currentNode.Content = keptNodes
+
+	return nil
+}

pkg/workflow_template.go

@@ -5,9 +5,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/onepanelio/core/pkg/util/extensions"
 	"github.com/onepanelio/core/pkg/util/ptr"
 	"github.com/onepanelio/core/pkg/util/request"
 	pagination "github.com/onepanelio/core/pkg/util/request/pagination"
+	yaml3 "gopkg.in/yaml.v3"
 	"strconv"
 	"strings"
 	"time"
@@ -65,6 +67,87 @@ func (c *Client) replaceSysNodePoolOptions(parameters []Parameter) (result []Par
 	return
 }
 
+func parameterOptionToNodes(option *ParameterOption) *yaml3.Node {
+	result := &yaml3.Node{
+		Kind: yaml3.MappingNode,
+	}
+
+	result.Content = append(result.Content, &yaml3.Node{
+		Kind:  yaml3.ScalarNode,
+		Value: "name",
+	})
+
+	result.Content = append(result.Content, &yaml3.Node{
+		Kind:  yaml3.ScalarNode,
+		Value: option.Name,
+	})
+
+	result.Content = append(result.Content, &yaml3.Node{
+		Kind:  yaml3.ScalarNode,
+		Value: "value",
+	})
+
+	result.Content = append(result.Content, &yaml3.Node{
+		Kind:  yaml3.ScalarNode,
+		Value: option.Value,
+	})
+
+	return result
+}
+
+func parameterOptionsToNodes(options []*ParameterOption) []*yaml3.Node {
+	result := make([]*yaml3.Node, 0)
+
+	for _, option := range options {
+		result = append(result, parameterOptionToNodes(option))
+	}
+
+	return result
+}
+
+// formatWorkflowTemplateManifest will remove any extraneous values from the workflow template manifest.
+// For example, select.nodepool should not have any options. If it does, they are stripped out.
+func formatWorkflowTemplateManifest(manifest string) (string, error) {
+	root := &yaml3.Node{}
+	err := yaml3.Unmarshal([]byte(manifest), root)
+	if err != nil {
+		return "", err
+	}
+
+	parametersIndex := extensions.CreateYamlIndex("arguments", "parameters")
+
+	if extensions.HasNode(root, parametersIndex) {
+		resultNode, err := extensions.GetNode(root, parametersIndex)
+		if err != nil {
+			return "", err
+		}
+
+		for _, child := range resultNode.Content {
+			hasKey, err := extensions.HasKeyValue(child, "type", "select.nodepool")
+			if err != nil {
+				return "", err
+			}
+
+			if hasKey {
+				if err := extensions.SetKeyValue(child, "value", "default"); err != nil {
+					return "", err
+				}
+
+				if err := extensions.DeleteNode(child, extensions.CreateYamlIndex("options")); err != nil {
+					return "", err
+				}
+			}
+		}
+	}
+
+	finalManifest, err := yaml3.Marshal(root)
+	if err != nil {
+		return "", err
+	}
+
+	return string(finalManifest), nil
+}
+
 func applyWorkflowTemplateFilter(sb sq.SelectBuilder, request *request.Request) (sq.SelectBuilder, error) {
 	if !request.HasFilter() {
 		return sb, nil
@@ -201,6 +284,12 @@ func (c *Client) createWorkflowTemplate(namespace string, workflowTemplate *Work
 		return nil, nil, err
 	}
 
+	newManifest, err := formatWorkflowTemplateManifest(workflowTemplate.Manifest)
+	if err != nil {
+		return nil, nil, err
+	}
+	workflowTemplate.Manifest = newManifest
+
 	params, err := ParseParametersFromManifest([]byte(workflowTemplate.Manifest))
 	if err != nil {
 		return nil, nil, util.NewUserError(codes.InvalidArgument, err.Error())
@@ -578,6 +667,12 @@ func (c *Client) CreateWorkflowTemplateVersion(namespace string, workflowTemplat
 		return nil, fmt.Errorf("uid required for CreateWorkflowTemplateVersion")
 	}
 
+	newManifest, err := formatWorkflowTemplateManifest(workflowTemplate.Manifest)
+	if err != nil {
+		return nil, err
+	}
+	workflowTemplate.Manifest = newManifest
+
 	// validate workflow template
 	if err := c.validateWorkflowTemplate(namespace, workflowTemplate); err != nil {
 		return nil, util.NewUserError(codes.InvalidArgument, err.Error())
@@ -1101,36 +1196,65 @@ func (c *Client) GetWorkflowTemplateLabels(namespace, name, prefix string, versi
 
 // GenerateWorkflowTemplateManifest replaces any special parameters with runtime values
 func (c *Client) GenerateWorkflowTemplateManifest(manifest string) (string, error) {
-	manifestObject := make(map[string]interface{})
-	if err := yaml.Unmarshal([]byte(manifest), &manifestObject); err != nil {
-		return "", util.NewUserError(codes.InvalidArgument, "Invalid yaml")
-	}
-
-	argumentsRaw := manifestObject["arguments"]
-	arguments, ok := argumentsRaw.(map[string]interface{})
-	if !ok {
-		return "", fmt.Errorf("unable to parse arguments")
-	}
-
-	jsonParameters, err := json.Marshal(arguments["parameters"])
-	if err != nil {
+	root := &yaml3.Node{}
+	if err := yaml3.Unmarshal([]byte(manifest), root); err != nil {
 		return "", err
 	}
 
-	parameters := make([]Parameter, 0)
-	if err := json.Unmarshal(jsonParameters, &parameters); err != nil {
-		return "", err
+	parametersIndex := extensions.CreateYamlIndex("arguments", "parameters")
+
+	if extensions.HasNode(root, parametersIndex) {
+		resultNode, err := extensions.GetNode(root, extensions.CreateYamlIndex("arguments", "parameters"))
+		if err != nil {
+			return "", err
+		}
+
+		nodePoolOptions, err := c.systemConfig.NodePoolOptions()
+		if err != nil {
+			return "", err
+		}
+
+		nodePoolParameterOptions := make([]*ParameterOption, 0)
+		for _, option := range nodePoolOptions {
+			nodePoolParameterOptions = append(nodePoolParameterOptions, &ParameterOption{
+				Name:  option.Name,
+				Value: option.Value,
+			})
+		}
+
+		for _, child := range resultNode.Content {
+			hasKey, err := extensions.HasKeyValue(child, "type", "select.nodepool")
+			if err != nil {
+				return "", err
+			}
+
+			if hasKey {
+				if err := extensions.SetKeyValue(child, "value", nodePoolParameterOptions[0].Value); err != nil {
+					return "", err
+				}
+
+				optionsIndex := extensions.CreateYamlIndex("options")
+				if !extensions.HasNode(child, optionsIndex) {
+					child.Content = append(child.Content, &yaml3.Node{
+						Kind:  yaml3.ScalarNode,
+						Value: "options",
+					}, &yaml3.Node{
+						Kind: yaml3.SequenceNode,
+					})
+				}
+
+				optionsNode, err := extensions.GetNode(child, optionsIndex)
+				if err != nil {
+					return "", err
+				}
+
+				optionsNode.Kind = yaml3.SequenceNode
+				optionsNode.Content = parameterOptionsToNodes(nodePoolParameterOptions)
+			}
+		}
 	}
 
-	parameters, err = c.replaceSysNodePoolOptions(parameters)
-	if err != nil {
-		return "", err
-	}
-
-	arguments["parameters"] = parameters
-	manifestObject["arguments"] = arguments
-
-	finalManifest, err := yaml.Marshal(manifestObject)
+	finalManifest, err := yaml3.Marshal(root)
 	if err != nil {
 		return "", err
 	}