Merge pull request #865 from Vafilor/feat/workspace.updates

feat: workspace updates

CONTRIBUTING.md

@@ -5,10 +5,10 @@
 Note: Up migrations are automatically executed when the application is run.
 
 ```bash
-docker run --rm --mount type=bind,source="${PWD}",target=/root onepanel-helper:v1.0.0 goose -dir db/sql create <name> sql  # Create migration in db/sql folder
-docker run --rm --mount type=bind,source="${PWD}",target=/root onepanel-helper:v1.0.0 goose -dir db postgres "${DB_DATASOURCE_NAME}" up # Migrate the DB to the most recent version available
-docker run --rm --mount type=bind,source="${PWD}",target=/root onepanel-helper:v1.0.0 goose -dir db postgres "${DB_DATASOURCE_NAME}" down # Roll back the version by 1
-docker run --rm --mount type=bind,source="${PWD}",target=/root onepanel-helper:v1.0.0 goose help  # See all available commands
+docker run --rm --mount type=bind,source="${PWD}",target=/root onepanel/helper:v1.0.0 goose -dir db/sql create <name> sql  # Create migration in db/sql folder
+docker run --rm --mount type=bind,source="${PWD}",target=/root onepanel/helper:v1.0.0 goose -dir db postgres "${DB_DATASOURCE_NAME}" up # Migrate the DB to the most recent version available
+docker run --rm --mount type=bind,source="${PWD}",target=/root onepanel/helper:v1.0.0 goose -dir db postgres "${DB_DATASOURCE_NAME}" down # Roll back the version by 1
+docker run --rm --mount type=bind,source="${PWD}",target=/root onepanel/helper:v1.0.0 goose help  # See all available commands
 ```
 
 ### Local 

api/api.swagger.json

@@ -2602,6 +2602,13 @@
             "in": "path",
             "required": true,
             "type": "string"
+          },
+          {
+            "name": "sinceTime",
+            "in": "query",
+            "required": false,
+            "type": "string",
+            "format": "int64"
           }
         ],
         "tags": [
@@ -3982,6 +3989,23 @@
           "items": {
             "$ref": "#/definitions/Parameter"
           }
+        },
+        "workspaceComponents": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/WorkspaceComponent"
+          }
+        }
+      }
+    },
+    "WorkspaceComponent": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
         }
       }
     },

api/gen/workspace.pb.gw.go

@@ -757,6 +757,10 @@ func local_request_WorkspaceService_RetryLastWorkspaceAction_0(ctx context.Conte
 
 }
 
+var (
+	filter_WorkspaceService_GetWorkspaceContainerLogs_0 = &utilities.DoubleArray{Encoding: map[string]int{"namespace": 0, "uid": 1, "containerName": 2}, Base: []int{1, 1, 2, 3, 0, 0, 0}, Check: []int{0, 1, 1, 1, 2, 3, 4}}
+)
+
 func request_WorkspaceService_GetWorkspaceContainerLogs_0(ctx context.Context, marshaler runtime.Marshaler, client WorkspaceServiceClient, req *http.Request, pathParams map[string]string) (WorkspaceService_GetWorkspaceContainerLogsClient, runtime.ServerMetadata, error) {
 	var protoReq GetWorkspaceContainerLogsRequest
 	var metadata runtime.ServerMetadata
@@ -798,6 +802,13 @@ func request_WorkspaceService_GetWorkspaceContainerLogs_0(ctx context.Context, m
 		return nil, metadata, status.Errorf(codes.InvalidArgument, "type mismatch, parameter: %s, error: %v", "containerName", err)
 	}
 
+	if err := req.ParseForm(); err != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+	if err := runtime.PopulateQueryParameters(&protoReq, req.Form, filter_WorkspaceService_GetWorkspaceContainerLogs_0); err != nil {
+		return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
+	}
+
 	stream, err := client.GetWorkspaceContainerLogs(ctx, &protoReq)
 	if err != nil {
 		return nil, metadata, err

api/proto/workspace.proto

@@ -81,6 +81,16 @@ service WorkspaceService {
 	}
 }
 
+message WorkspaceComponent {
+	string name = 1;
+	string url = 2;
+}
+
+message WorkspaceComponent {
+	string name = 1;
+	string url = 2;
+}
+
 message Workspace {
 	string uid = 1;
 	string name = 2;
@@ -92,6 +102,7 @@ message Workspace {
 	repeated KeyValue labels = 8;
 	string url = 9;
 	repeated Parameter templateParameters = 10;
+	repeated WorkspaceComponent workspaceComponents = 11;
 }
 
 message WorkspaceStatus {
@@ -205,4 +216,5 @@ message GetWorkspaceContainerLogsRequest {
 	string namespace = 1;
 	string uid = 2;
 	string containerName = 3;
+	int64 sinceTime = 4;
 }

pkg/config.go

@@ -62,6 +62,7 @@ func (c *Client) GetDefaultConfig() (config *ConfigMap, err error) {
 	return
 }
 
+// GetNamespaceConfig returns the NamespaceConfig given a namespace
 func (c *Client) GetNamespaceConfig(namespace string) (config *NamespaceConfig, err error) {
 	configMap, err := c.getConfigMap(namespace, "onepanel")
 	if err != nil {

pkg/config_types.go

@@ -350,6 +350,7 @@ func (g *ArtifactRepositoryGCSProvider) FormatKey(namespace, workflowName, podNa
 	return keyFormat
 }
 
+// NamespaceConfig represents configuration for the namespace
 type NamespaceConfig struct {
 	ArtifactRepository ArtifactRepositoryProvider
 }

pkg/util/extensions/extensions.go

@@ -145,6 +145,24 @@ func HasKeyValue(node *yaml.Node, key string, values ...string) (bool, error) {
 	return false, nil
 }
 
+// GetKeyValue gets the value of the key from the node (assumed to be a mapping node)
+func GetKeyValue(node *yaml.Node, key string) (*yaml.Node, error) {
+	if node.Kind != yaml.MappingNode {
+		return nil, fmt.Errorf("not a mapping node")
+	}
+
+	for i := 0; i < len(node.Content)-1; i += 2 {
+		keyNode := node.Content[i]
+		valueNode := node.Content[i+1]
+
+		if keyNode.Value == key {
+			return valueNode, nil
+		}
+	}
+
+	return nil, fmt.Errorf("not found")
+}
+
 // Iterate runs through all of the content nodes in the indicated root node
 func Iterate(root *yaml.Node, callable func(parent, value *yaml.Node)) {
 	for _, child := range root.Content {

pkg/workflow_execution.go

@@ -1683,7 +1683,9 @@ func (c *Client) ListFiles(namespace, key string) (files []*File, err error) {
 
 	files = make([]*File, 0)
 
-	if len(key) > 0 {
+	if key == "/" {
+		key = ""
+	} else if len(key) > 0 {
 		if string(key[len(key)-1]) != "/" {
 			key += "/"
 		}

pkg/workspace.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	sq "github.com/Masterminds/squirrel"
 	wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
@@ -829,18 +830,22 @@ func (c *Client) updateWorkspace(namespace, uid, workspaceAction, resourceAction
 	return
 }
 
+// UpdateWorkspace marks a workspace as "updating"
 func (c *Client) UpdateWorkspace(namespace, uid string, parameters []Parameter) (err error) {
 	return c.updateWorkspace(namespace, uid, "update", "apply", &WorkspaceStatus{Phase: WorkspaceUpdating}, parameters...)
 }
 
+// PauseWorkspace pauses a workspace
 func (c *Client) PauseWorkspace(namespace, uid string) (err error) {
 	return c.updateWorkspace(namespace, uid, "pause", "delete", &WorkspaceStatus{Phase: WorkspacePausing})
 }
 
+// ResumeWorkspace resumes a workspace
 func (c *Client) ResumeWorkspace(namespace, uid string) (err error) {
 	return c.updateWorkspace(namespace, uid, "create", "apply", &WorkspaceStatus{Phase: WorkspaceLaunching})
 }
 
+// DeleteWorkspace deletes a workspace
 func (c *Client) DeleteWorkspace(namespace, uid string) (err error) {
 	return c.updateWorkspace(namespace, uid, "delete", "delete", &WorkspaceStatus{Phase: WorkspaceTerminating})
 }
@@ -883,13 +888,16 @@ func (c *Client) GetWorkspaceStatisticsForNamespace(namespace string) (report *W
 }
 
 // GetWorkspaceContainerLogs returns logs for a given container name in a Workspace
-func (c *Client) GetWorkspaceContainerLogs(namespace, uid, containerName string) (<-chan []*LogEntry, error) {
+func (c *Client) GetWorkspaceContainerLogs(namespace, uid, containerName string, sinceTime time.Time) (<-chan []*LogEntry, error) {
 	var stream io.ReadCloser
 
+	k8sSinceTime := metav1.NewTime(sinceTime)
+
 	stream, err := c.CoreV1().Pods(namespace).GetLogs(uid+"-0", &corev1.PodLogOptions{
 		Container:  containerName,
 		Follow:     true,
 		Timestamps: true,
+		SinceTime:  &k8sSinceTime,
 	}).Stream()
 
 	if err != nil {

pkg/workspace_template.go

@@ -988,7 +988,7 @@ func (c *Client) generateWorkspaceTemplateWorkflowTemplate(workspaceTemplate *Wo
 	return workflowTemplate, nil
 }
 
-// CreateWorkspaceTemplateWorkflowTemplate generates and returns a workflowTemplate for a given workspaceTemplate manifest
+// GenerateWorkspaceTemplateWorkflowTemplate generates and returns a workflowTemplate for a given workspaceTemplate manifest
 func (c *Client) GenerateWorkspaceTemplateWorkflowTemplate(workspaceTemplate *WorkspaceTemplate) (workflowTemplate *WorkflowTemplate, err error) {
 	workflowTemplate, err = c.generateWorkspaceTemplateWorkflowTemplate(workspaceTemplate)
 	if err != nil {

pkg/workspace_template_types.go

@@ -3,13 +3,24 @@ package v1
 import (
 	"fmt"
 	wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
+	"github.com/onepanelio/core/pkg/util/extensions"
 	"github.com/onepanelio/core/pkg/util/sql"
 	"github.com/onepanelio/core/pkg/util/types"
 	uid2 "github.com/onepanelio/core/pkg/util/uid"
+	yaml3 "gopkg.in/yaml.v3"
 	"sigs.k8s.io/yaml"
+	"strings"
 	"time"
 )
 
+// WorkspaceService represents services available to external access in a Workspace
+type WorkspaceService struct {
+	Name string
+	Path string
+}
+
+// WorkspaceTemplate represents the data associated with a WorkspaceTemplate
+// this is a mix of DB and "in-memory" fields
 type WorkspaceTemplate struct {
 	ID                         uint64
 	WorkspaceTemplateVersionID uint64 `db:"workspace_template_version_id"`
@@ -80,6 +91,58 @@ func (wt *WorkspaceTemplate) InjectRuntimeParameters(config SystemConfig) error
 	return nil
 }
 
+// GetServices returns an array of WorkspaceServices
+func (wt *WorkspaceTemplate) GetServices() ([]*WorkspaceService, error) {
+	result := make([]*WorkspaceService, 0)
+
+	root := &yaml3.Node{}
+	if err := yaml3.Unmarshal([]byte(wt.Manifest), root); err != nil {
+		return nil, err
+	}
+
+	containers, err := extensions.GetNode(root, extensions.CreateYamlIndex("containers"))
+	if err != nil {
+		if strings.Contains(err.Error(), "not found") {
+			return result, nil
+		}
+
+		return nil, err
+	}
+
+	for _, container := range containers.Content {
+		hasKeyValue, err := extensions.HasKeyValue(container, "name", "sys-filesyncer")
+		if err != nil {
+			return nil, err
+		}
+
+		if hasKeyValue {
+			argsValue, err := extensions.GetKeyValue(container, "args")
+			if err != nil {
+				continue
+			}
+
+			path := ""
+			for _, arg := range argsValue.Content {
+				if strings.Contains(arg.Value, "server-prefix") {
+					parts := strings.Split(arg.Value, "=")
+					if len(parts) > 1 {
+						path = parts[1]
+					}
+				}
+			}
+
+			fmt.Printf("%v", argsValue)
+
+			result = append(result, &WorkspaceService{
+				Name: "sys-filesyncer",
+				Path: path,
+			})
+		}
+	}
+
+	return result, nil
+}
+
 // WorkspaceTemplatesToVersionIDs plucks the WorkspaceTemplateVersionID from each template and returns it in an array
 // No duplicates are included.
 func WorkspaceTemplatesToVersionIDs(resources []*WorkspaceTemplate) (ids []uint64) {

server/workspace_server.go

@@ -32,6 +32,10 @@ func NewWorkspaceServer() *WorkspaceServer {
 }
 
 func apiWorkspace(wt *v1.Workspace, config v1.SystemConfig) *api.Workspace {
+	if wt == nil {
+		return nil
+	}
+
 	protocol := config.APIProtocol()
 	domain := config.Domain()
 
@@ -53,11 +57,25 @@ func apiWorkspace(wt *v1.Workspace, config v1.SystemConfig) *api.Workspace {
 		return nil
 	}
 
+	services, err := wt.WorkspaceTemplate.GetServices()
+	if err != nil {
+		return nil
+	}
+
+	apiServices := make([]*api.WorkspaceComponent, 0)
+	for _, service := range services {
+		apiServices = append(apiServices, &api.WorkspaceComponent{
+			Name: service.Name,
+			Url:  wt.GetURL(*protocol, *domain) + service.Path,
+		})
+	}
+
 	res := &api.Workspace{
-		Uid:       wt.UID,
-		Name:      wt.Name,
-		CreatedAt: wt.CreatedAt.UTC().Format(time.RFC3339),
-		Url:       wt.GetURL(*protocol, *domain),
+		Uid:                 wt.UID,
+		Name:                wt.Name,
+		CreatedAt:           wt.CreatedAt.UTC().Format(time.RFC3339),
+		Url:                 wt.GetURL(*protocol, *domain),
+		WorkspaceComponents: apiServices,
 	}
 	res.Parameters = converter.ParametersToAPI(wt.Parameters)
 
@@ -88,6 +106,7 @@ func apiWorkspace(wt *v1.Workspace, config v1.SystemConfig) *api.Workspace {
 	return res
 }
 
+// CreateWorkspace create a workspace
 func (s *WorkspaceServer) CreateWorkspace(ctx context.Context, req *api.CreateWorkspaceRequest) (*api.Workspace, error) {
 	client := getClient(ctx)
 	allowed, err := auth.IsAuthorized(client, req.Namespace, "create", "onepanel.io", "workspaces", "")
@@ -137,6 +156,7 @@ func (s *WorkspaceServer) CreateWorkspace(ctx context.Context, req *api.CreateWo
 	return apiWorkspace, nil
 }
 
+// GetWorkspace returns a Workspace information
 func (s *WorkspaceServer) GetWorkspace(ctx context.Context, req *api.GetWorkspaceRequest) (*api.Workspace, error) {
 	client := getClient(ctx)
 	allowed, err := auth.IsAuthorized(client, req.Namespace, "get", "onepanel.io", "workspaces", req.Uid)
@@ -148,6 +168,9 @@ func (s *WorkspaceServer) GetWorkspace(ctx context.Context, req *api.GetWorkspac
 	if err != nil {
 		return nil, err
 	}
+	if workspace == nil {
+		return nil, util.NewUserError(codes.NotFound, "Not found")
+	}
 
 	sysConfig, err := client.GetSystemConfig()
 	if err != nil {
@@ -173,6 +196,7 @@ func (s *WorkspaceServer) GetWorkspace(ctx context.Context, req *api.GetWorkspac
 	return apiWorkspace, nil
 }
 
+// UpdateWorkspaceStatus updates a given workspaces status such as running, paused, etc.
 func (s *WorkspaceServer) UpdateWorkspaceStatus(ctx context.Context, req *api.UpdateWorkspaceStatusRequest) (*empty.Empty, error) {
 	client := getClient(ctx)
 	allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "onepanel.io", "workspaces", req.Uid)
@@ -189,6 +213,7 @@ func (s *WorkspaceServer) UpdateWorkspaceStatus(ctx context.Context, req *api.Up
 	return &empty.Empty{}, err
 }
 
+// UpdateWorkspace updates a workspace's status
 func (s *WorkspaceServer) UpdateWorkspace(ctx context.Context, req *api.UpdateWorkspaceRequest) (*empty.Empty, error) {
 	client := getClient(ctx)
 	allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "onepanel.io", "workspaces", req.Uid)
@@ -212,6 +237,7 @@ func (s *WorkspaceServer) UpdateWorkspace(ctx context.Context, req *api.UpdateWo
 	return &empty.Empty{}, err
 }
 
+// ListWorkspaces lists the current workspaces for a given namespace
 func (s *WorkspaceServer) ListWorkspaces(ctx context.Context, req *api.ListWorkspaceRequest) (*api.ListWorkspaceResponse, error) {
 	client := getClient(ctx)
 	allowed, err := auth.IsAuthorized(client, req.Namespace, "list", "onepanel.io", "workspaces", "")
@@ -272,6 +298,7 @@ func (s *WorkspaceServer) ListWorkspaces(ctx context.Context, req *api.ListWorks
 	}, nil
 }
 
+// PauseWorkspace requests to pause a given workspace
 func (s *WorkspaceServer) PauseWorkspace(ctx context.Context, req *api.PauseWorkspaceRequest) (*empty.Empty, error) {
 	client := getClient(ctx)
 	allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "onepanel.io", "workspaces", req.Uid)
@@ -284,6 +311,7 @@ func (s *WorkspaceServer) PauseWorkspace(ctx context.Context, req *api.PauseWork
 	return &empty.Empty{}, err
 }
 
+// ResumeWorkspace attempts to resume a workspace
 func (s *WorkspaceServer) ResumeWorkspace(ctx context.Context, req *api.ResumeWorkspaceRequest) (*empty.Empty, error) {
 	client := getClient(ctx)
 	allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "onepanel.io", "workspaces", req.Uid)
@@ -296,6 +324,7 @@ func (s *WorkspaceServer) ResumeWorkspace(ctx context.Context, req *api.ResumeWo
 	return &empty.Empty{}, err
 }
 
+// DeleteWorkspace requests to delete a workspace
 func (s *WorkspaceServer) DeleteWorkspace(ctx context.Context, req *api.DeleteWorkspaceRequest) (*empty.Empty, error) {
 	client := getClient(ctx)
 	allowed, err := auth.IsAuthorized(client, req.Namespace, "delete", "onepanel.io", "workspaces", req.Uid)
@@ -396,7 +425,8 @@ func (s *WorkspaceServer) GetWorkspaceContainerLogs(req *api.GetWorkspaceContain
 		return err
 	}
 
-	watcher, err := client.GetWorkspaceContainerLogs(req.Namespace, req.Uid, req.ContainerName)
+	sinceTime := time.Unix(req.SinceTime, 0)
+	watcher, err := client.GetWorkspaceContainerLogs(req.Namespace, req.Uid, req.ContainerName, sinceTime)
 	if err != nil {
 		return err
 	}