Merge pull request #321 from onepanelio/dev

chore: dev > master  (v0.10.0)

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+<!--  Thanks for sending a pull request! Here are some tips for you:
+1. Please read our contributor guidelines: https://docs.onepanel.ai/docs/getting-started/contributing
+2. Prefix the title of this PR with `feat:`, `fix:`, `docs:` or `chore:`, example: `feat: added great feature`
+3. If this PR is a feature or enhancement, then create an issue (https://github.com/onepanelio/core/issues) first. 
+-->
+
+**What this PR does**:
+
+**Which issue(s) this PR fixes**:
+<!--
+*Automatically closes linked issue when PR is merged.
+Usage: `Fixes onepanelio/core#<issue-number>`
+-->
+Fixes onepanelio/core#
+
+**Special notes for your reviewer**:

.github/semantic.yml

@@ -0,0 +1 @@
+titleOnly: true

.github/workflows/push_dev_branch.yaml

@@ -1,8 +1,8 @@
-name: Publish develop docker image
+name: Publish dev docker image
 on:
   push:
     branches:
-      - develop
+      - dev
 jobs:
   build:
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -49,7 +49,7 @@ Make sure that your `$GOBIN` is in your `$PATH`.
 
 Generate Go and Swagger APIs:
 ```bash
-make api
+make api version=1.0.0
 ```
 
 ## Minikube Debugging and Development
@@ -122,6 +122,8 @@ After this, build main.go and run the executable.
 
 ## Code Structure & Organization
 
+### `utils` dir
+
 ```shell script
 utils/*.go
 ```
@@ -150,4 +152,25 @@ These can be pulled out into their own package or into a new v2 directory if nee
 You can add
 - kubernetes specific operations
 - database specific operations
-- types
+- types
+
+### `cmd` dir
+Each source file here is assumed to result in an executable.
+- Hence the `package main` at the top of each
+
+Place each source file into it's own folder.
+Example source file name: `flush_cache.go`
+- Dir structure: `cmd/flush-cache/flush_cache.go`
+
+To avoid errors like this during docker build
+```text
+# github.com/onepanelio/core/cmd
+cmd/goose.go:22:6: main redeclared in this block
+        previous declaration at cmd/gen-release-md.go:136:6
+github.com/onepanelio/core
+```
+Caused by
+```dockerfile
+RUN go install -v ./...
+```
+

Dockerfile

@@ -5,11 +5,13 @@ COPY . .
 
 RUN go get -d -v ./...
 RUN go install -v ./...
-
+RUN go get -u github.com/pressly/goose/cmd/goose
+RUN go build -o /go/bin/goose ./cmd/goose/goose.go
 
 FROM golang:1.13.10
 COPY --from=builder /go/bin/core .
 COPY --from=builder /go/src/db ./db
+COPY --from=builder /go/bin/goose .
 
 EXPOSE 8888
 EXPOSE 8887

LICENSE

@@ -1,3 +1,5 @@
+                Copyright 2020 Onepanel, Inc.  All rights reserved.
+                
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/

Makefile

@@ -1,9 +1,17 @@
 COMMIT_HASH=$(shell git rev-parse --short HEAD)
 
+.PHONY: init
+
+init:
+ifndef version
+	$(error version is undefined)
+endif
+
 jq:
 	cat api/apidocs.swagger.json \
 		| jq 'walk( if type == "object" then with_entries( .key |= sub( "api\\."; "") ) else . end )' \
 		| jq 'walk( if type == "string" then gsub( "api\\."; "") else . end )' \
+		| jq '.info.version = "$(version)"' \
 		> api/api.swagger.json
 	rm api/apidocs.swagger.json
 
@@ -16,7 +24,7 @@ protoc:
  		--grpc-gateway_out=logtostderr=true,allow_delete_body=true:api \
  		--swagger_out=allow_merge=true,fqn_for_swagger_name=true,allow_delete_body=true,logtostderr=true,simple_operation_ids=true:api
 
-api: protoc jq
+api: init protoc jq
 
 docker-build:
 	docker build -t onepanel-core .

README.md

@@ -1,3 +1,5 @@
+![Build and publish to Docker Hub](https://github.com/onepanelio/core/workflows/Build%20and%20publish%20to%20Docker%20Hub/badge.svg)
+
 # Onepanel
 
 Welcome to Onepanel! This is the main repository for the API. It is also where you can submit bugs and enhancement requests.
@@ -10,4 +12,4 @@ See our [Quick start guide](https://docs.onepanel.ai/docs/getting-started/quicks
 See our [Contribution guide](https://docs.onepanel.ai/docs/getting-started/contributing) to get started.
 
 ## Acknowledgments
-Onepanel uses the excellent [Argo](https://github.com/argoproj/argo) project under the hood to orchestrate workflows.
+Onepanel uses the excellent [Argo](https://github.com/argoproj/argo) project under the hood to orchestrate workflows.

api/api.pb.go

@@ -31,25 +31,23 @@ var file_api_proto_rawDesc = []byte{
 	0x0a, 0x09, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x61, 0x70, 0x69,
 	0x1a, 0x2c, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x2d, 0x67, 0x65, 0x6e, 0x2d, 0x73, 0x77, 0x61,
 	0x67, 0x67, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x61, 0x6e, 0x6e,
-	0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x8b,
-	0x02, 0x92, 0x41, 0x87, 0x02, 0x12, 0x74, 0x0a, 0x0d, 0x4f, 0x6e, 0x65, 0x70, 0x61, 0x6e, 0x65,
-	0x6c, 0x20, 0x43, 0x6f, 0x72, 0x65, 0x12, 0x19, 0x4f, 0x6e, 0x65, 0x70, 0x61, 0x6e, 0x65, 0x6c,
-	0x20, 0x43, 0x6f, 0x72, 0x65, 0x20, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x20, 0x41, 0x50,
-	0x49, 0x22, 0x3b, 0x0a, 0x15, 0x4f, 0x6e, 0x65, 0x70, 0x61, 0x6e, 0x65, 0x6c, 0x20, 0x43, 0x6f,
-	0x72, 0x65, 0x20, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x22, 0x68, 0x74, 0x74, 0x70,
-	0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f,
-	0x6e, 0x65, 0x70, 0x61, 0x6e, 0x65, 0x6c, 0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x32, 0x0b,
-	0x31, 0x2e, 0x30, 0x2e, 0x30, 0x2d, 0x62, 0x65, 0x74, 0x61, 0x31, 0x1a, 0x0e, 0x6c, 0x6f, 0x63,
-	0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x3a, 0x38, 0x38, 0x38, 0x38, 0x2a, 0x02, 0x01, 0x02, 0x32,
-	0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f,
-	0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a,
-	0x73, 0x6f, 0x6e, 0x5a, 0x49, 0x0a, 0x47, 0x0a, 0x06, 0x42, 0x65, 0x61, 0x72, 0x65, 0x72, 0x12,
-	0x3d, 0x08, 0x02, 0x12, 0x28, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74,
-	0x69, 0x6f, 0x6e, 0x20, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x2c, 0x20, 0x70, 0x72, 0x65, 0x66, 0x69,
-	0x78, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x42, 0x65, 0x61, 0x72, 0x65, 0x72, 0x1a, 0x0d, 0x61,
-	0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x02, 0x62, 0x0c,
-	0x0a, 0x0a, 0x0a, 0x06, 0x42, 0x65, 0x61, 0x72, 0x65, 0x72, 0x12, 0x00, 0x62, 0x06, 0x70, 0x72,
-	0x6f, 0x74, 0x6f, 0x33,
+	0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0xef,
+	0x01, 0x92, 0x41, 0xeb, 0x01, 0x12, 0x58, 0x0a, 0x08, 0x4f, 0x6e, 0x65, 0x70, 0x61, 0x6e, 0x65,
+	0x6c, 0x12, 0x0c, 0x4f, 0x6e, 0x65, 0x70, 0x61, 0x6e, 0x65, 0x6c, 0x20, 0x41, 0x50, 0x49, 0x22,
+	0x36, 0x0a, 0x10, 0x4f, 0x6e, 0x65, 0x70, 0x61, 0x6e, 0x65, 0x6c, 0x20, 0x70, 0x72, 0x6f, 0x6a,
+	0x65, 0x63, 0x74, 0x12, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74,
+	0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x6e, 0x65, 0x70, 0x61, 0x6e, 0x65, 0x6c,
+	0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x32, 0x06, 0x30, 0x2e, 0x31, 0x30, 0x2e, 0x30, 0x1a,
+	0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x3a, 0x38, 0x38, 0x38, 0x38, 0x2a,
+	0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x5a, 0x49, 0x0a, 0x47, 0x0a, 0x06, 0x42, 0x65, 0x61,
+	0x72, 0x65, 0x72, 0x12, 0x3d, 0x08, 0x02, 0x12, 0x28, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74,
+	0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x2c, 0x20, 0x70,
+	0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x42, 0x65, 0x61, 0x72, 0x65,
+	0x72, 0x1a, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x20, 0x02, 0x62, 0x0c, 0x0a, 0x0a, 0x0a, 0x06, 0x42, 0x65, 0x61, 0x72, 0x65, 0x72, 0x12, 0x00,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var file_api_proto_goTypes = []interface{}{}

api/api.proto

@@ -6,11 +6,11 @@ import "protoc-gen-swagger/options/annotations.proto";
 
 option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = {
     info: {
-        title: "Onepanel Core";
-        description: "Onepanel Core project API";
-        version: "1.0.0-beta1";
+        title: "Onepanel";
+        description: "Onepanel API";
+        version: "0.10.0";
 		contact: {
-			name: "Onepanel Core project";
+			name: "Onepanel project";
 			url: "https://github.com/onepanelio/core";
 		};
     };

api/api.swagger.json

@@ -1,11 +1,11 @@
 {
   "swagger": "2.0",
   "info": {
-    "title": "Onepanel Core",
-    "description": "Onepanel Core project API",
-    "version": "1.0.0-beta1",
+    "title": "Onepanel",
+    "description": "Onepanel API",
+    "version": "0.10.0",
     "contact": {
-      "name": "Onepanel Core project",
+      "name": "Onepanel project",
       "url": "https://github.com/onepanelio/core"
     }
   },
@@ -3167,6 +3167,9 @@
         "createdAt": {
           "type": "string"
         },
+        "modifiedAt": {
+          "type": "string"
+        },
         "uid": {
           "type": "string"
         },

api/workflow_template.pb.go

@@ -768,16 +768,17 @@ type WorkflowTemplate struct {
 	unknownFields protoimpl.UnknownFields
 
 	CreatedAt  string                            `protobuf:"bytes,1,opt,name=createdAt,proto3" json:"createdAt,omitempty"`
-	Uid        string                            `protobuf:"bytes,2,opt,name=uid,proto3" json:"uid,omitempty"`
-	Name       string                            `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"`
-	Version    int64                             `protobuf:"varint,4,opt,name=version,proto3" json:"version,omitempty"`
-	Versions   int64                             `protobuf:"varint,5,opt,name=versions,proto3" json:"versions,omitempty"`
-	Manifest   string                            `protobuf:"bytes,6,opt,name=manifest,proto3" json:"manifest,omitempty"`
-	IsLatest   bool                              `protobuf:"varint,7,opt,name=isLatest,proto3" json:"isLatest,omitempty"`
-	IsArchived bool                              `protobuf:"varint,8,opt,name=isArchived,proto3" json:"isArchived,omitempty"`
-	Labels     []*KeyValue                       `protobuf:"bytes,9,rep,name=labels,proto3" json:"labels,omitempty"`
-	Stats      *WorkflowExecutionStatisticReport `protobuf:"bytes,10,opt,name=stats,proto3" json:"stats,omitempty"`
-	CronStats  *CronWorkflowStatisticsReport     `protobuf:"bytes,11,opt,name=cronStats,proto3" json:"cronStats,omitempty"`
+	ModifiedAt string                            `protobuf:"bytes,2,opt,name=modifiedAt,proto3" json:"modifiedAt,omitempty"`
+	Uid        string                            `protobuf:"bytes,3,opt,name=uid,proto3" json:"uid,omitempty"`
+	Name       string                            `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"`
+	Version    int64                             `protobuf:"varint,5,opt,name=version,proto3" json:"version,omitempty"`
+	Versions   int64                             `protobuf:"varint,6,opt,name=versions,proto3" json:"versions,omitempty"`
+	Manifest   string                            `protobuf:"bytes,7,opt,name=manifest,proto3" json:"manifest,omitempty"`
+	IsLatest   bool                              `protobuf:"varint,8,opt,name=isLatest,proto3" json:"isLatest,omitempty"`
+	IsArchived bool                              `protobuf:"varint,9,opt,name=isArchived,proto3" json:"isArchived,omitempty"`
+	Labels     []*KeyValue                       `protobuf:"bytes,10,rep,name=labels,proto3" json:"labels,omitempty"`
+	Stats      *WorkflowExecutionStatisticReport `protobuf:"bytes,11,opt,name=stats,proto3" json:"stats,omitempty"`
+	CronStats  *CronWorkflowStatisticsReport     `protobuf:"bytes,12,opt,name=cronStats,proto3" json:"cronStats,omitempty"`
 }
 
 func (x *WorkflowTemplate) Reset() {
@@ -819,6 +820,13 @@ func (x *WorkflowTemplate) GetCreatedAt() string {
 	return ""
 }
 
+func (x *WorkflowTemplate) GetModifiedAt() string {
+	if x != nil {
+		return x.ModifiedAt
+	}
+	return ""
+}
+
 func (x *WorkflowTemplate) GetUid() string {
 	if x != nil {
 		return x.Uid
@@ -1051,29 +1059,31 @@ var file_workflow_template_proto_rawDesc = []byte{
 	0x74, 0x65, 0x64, 0x22, 0x34, 0x0a, 0x1c, 0x43, 0x72, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66,
 	0x6c, 0x6f, 0x77, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x70,
 	0x6f, 0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x05, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x89, 0x03, 0x0a, 0x10, 0x57, 0x6f,
+	0x28, 0x05, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0xa9, 0x03, 0x0a, 0x10, 0x57, 0x6f,
 	0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x1c,
 	0x0a, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x10, 0x0a, 0x03,
-	0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x12,
-	0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61,
-	0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20,
+	0x09, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1e, 0x0a, 0x0a,
+	0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x12, 0x10, 0x0a, 0x03,
+	0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x12,
+	0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61,
+	0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20,
 	0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08,
-	0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08,
+	0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08,
 	0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x61, 0x6e, 0x69,
-	0x66, 0x65, 0x73, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x61, 0x6e, 0x69,
+	0x66, 0x65, 0x73, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x61, 0x6e, 0x69,
 	0x66, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x73, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74,
-	0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74,
-	0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x73, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x64, 0x18, 0x08,
+	0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74,
+	0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x73, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x64, 0x18, 0x09,
 	0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x73, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x64,
-	0x12, 0x25, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b,
+	0x12, 0x25, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b,
 	0x32, 0x0d, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52,
 	0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x3b, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73,
-	0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x57, 0x6f, 0x72,
+	0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x57, 0x6f, 0x72,
 	0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74,
 	0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x05, 0x73,
 	0x74, 0x61, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x09, 0x63, 0x72, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74,
-	0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x72,
+	0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x72,
 	0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73,
 	0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x09, 0x63, 0x72, 0x6f, 0x6e,
 	0x53, 0x74, 0x61, 0x74, 0x73, 0x22, 0x6e, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b,

api/workflow_template.proto

@@ -135,18 +135,18 @@ message CronWorkflowStatisticsReport {
 
 message WorkflowTemplate {
     string createdAt = 1;
-    string uid = 2;
-    string name = 3;
-    int64 version = 4;
-    int64 versions = 5;
-    string manifest = 6;
-    bool isLatest = 7;
-    bool isArchived = 8;
-    repeated KeyValue labels = 9;
-
-    WorkflowExecutionStatisticReport stats = 10;
-    CronWorkflowStatisticsReport cronStats = 11;
+    string modifiedAt = 2;
+    string uid = 3;
+    string name = 4;
+    int64 version = 5;
+    int64 versions = 6;
+    string manifest = 7;
+    bool isLatest = 8;
+    bool isArchived = 9;
+    repeated KeyValue labels = 10;
 
+    WorkflowExecutionStatisticReport stats = 11;
+    CronWorkflowStatisticsReport cronStats = 12;
 }
 
 message GetWorkflowTemplateLabelsRequest {

cmd/README.md

@@ -0,0 +1,17 @@
+# Helper scripts
+
+## gen-release-md.go
+Generates markdown for releases.
+
+Usage:
+```bash
+go run cmd/gen-release-md/gen-release-md.go -v=0.10.0 > /tmp/release.md
+```
+
+## goose.go
+Supports both Go and SQL migrations.
+
+```bash
+go run cmd/goose/goose up     # run up migrations
+go run cmd/goose/goose down   # run down migrations
+```

cmd/gen-release-md/gen-release-md.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+type User struct {
+	Login     string `json:"login"`
+	URL       string `json:"html_url"`
+	AvatarURL string `json:"avatar_url"`
+}
+
+type Label struct {
+	Name string `json:"name"`
+}
+
+type PullRequest struct {
+	URL string `json:"url"`
+}
+
+type Issue struct {
+	Number      int          `json:"number"`
+	URL         string       `json:"html_url"`
+	Title       string       `json:"title"`
+	User        User         `json:"user"`
+	PullRequest *PullRequest `json:"pull_request"`
+	Labels      []Label      `json:"labels"`
+}
+
+const (
+	apiPrefix          = "https://api.github.com/repos/"
+	issuesPathAndQuery = "/issues?milestone=3&state=closed"
+)
+
+var releaseTemplate = `# Documentation
+See [Documentation](https://docs.onepanel.ai)
+
+# CLI Installation
+
+## Linux
+
+` + "```" + `
+# Download the binary
+curl -sLO https://github.com/onepanelio/core/releases/download/v%s/opctl-linux-amd64
+
+# Make binary executable
+chmod +x opctl-linux-amd64
+
+# Move binary to path
+mv ./opctl-linux-amd64 /usr/local/bin/opctl
+
+# Test installation
+opctl version
+` + "```" + `
+
+## macOS
+
+` + "```" + `
+# Download the binary
+curl -sLO https://github.com/onepanelio/core/releases/download/v%s/opctl-macos-amd64
+
+# Make binary executable
+chmod +x opctl-macos-amd64
+
+# Move binary to path
+mv ./opctl-macos-amd64 /usr/local/bin/opctl
+
+# Test installation
+opctl version
+` + "```"
+
+var repositories = []string{
+	"onepanelio/core",
+	"onepanelio/core-ui",
+	"onepanelio/cli",
+	"onepanelio/manifests",
+	"onepanelio/core-docs",
+}
+
+// Parse issues, pulling only PRs and categorize them based on labels
+// Print everything as MD that can be copied into release notes
+func printMarkDown(issues []*Issue, version *string) {
+	contributors := make(map[string]User, 0)
+	sections := make(map[string]string, 0)
+
+	for _, iss := range issues {
+		if iss.PullRequest == nil {
+			continue
+		}
+
+		parts := strings.Split(iss.Title, ":")
+		if len(parts) > 0 {
+			contributors[iss.User.Login] = iss.User
+			sections[parts[0]] += fmt.Sprintf("- %s ([#%d](%s))\n", iss.Title, iss.Number, iss.URL)
+		}
+	}
+
+	releaseTemplate := fmt.Sprintf(releaseTemplate, *version, *version)
+	fmt.Println(releaseTemplate)
+	fmt.Println("# Changelog\n")
+	fmt.Println("## Features\n")
+	fmt.Println(sections["feat"])
+	fmt.Println("## Fixes\n")
+	fmt.Println(sections["fix"])
+	fmt.Println("## Docs\n")
+	fmt.Println(sections["docs"])
+	fmt.Println("## Chores\n")
+	fmt.Println(sections["chore"])
+
+	fmt.Println("# Contributors\n")
+	for _, user := range contributors {
+		fmt.Println(fmt.Sprintf("- <a href=\"%s\"><img src=\"%s\" width=\"12\"/> <strong>%s</strong></a> %s", user.URL, user.AvatarURL, user.Login, user.Login))
+	}
+}
+
+// Get issues from repository
+func getIssues(repository string, username *string) ([]*Issue, error) {
+	client := &http.Client{}
+	req, err := http.NewRequest(http.MethodGet, apiPrefix+repository+issuesPathAndQuery, nil)
+	if username != nil {
+		req.SetBasicAuth(*username, "")
+	}
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer res.Body.Close()
+
+	issues := make([]*Issue, 0)
+	if err = json.NewDecoder(res.Body).Decode(&issues); err != nil {
+		return nil, err
+	}
+
+	return issues, nil
+}
+
+func main() {
+	version := flag.String("v", "1.0.0", "Version of release, example: -v=1.0.0")
+	username := flag.String("u", "", "GitHub username for request, example: -u=octocat")
+
+	flag.Parse()
+
+	issues := make([]*Issue, 0)
+	for _, repository := range repositories {
+		iss, err := getIssues(repository, username)
+		if err != nil {
+			return
+		}
+		issues = append(issues, iss...)
+	}
+
+	printMarkDown(issues, version)
+}

cmd/goose/goose.go

@@ -0,0 +1,56 @@
+// This is custom goose binary to support .go migration files in ./db dir
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/onepanelio/core/db"
+	v1 "github.com/onepanelio/core/pkg"
+	"log"
+	"os"
+
+	"github.com/pressly/goose"
+)
+
+var (
+	flags = flag.NewFlagSet("goose", flag.ExitOnError)
+	dir   = flags.String("dir", ".", "directory with migration files")
+)
+
+func main() {
+	flags.Parse(os.Args[1:])
+	args := flags.Args()
+
+	if len(args) < 1 {
+		flags.Usage()
+		return
+	}
+
+	kubeConfig := v1.NewConfig()
+	client, err := v1.NewClient(kubeConfig, nil)
+	if err != nil {
+		log.Fatalf("Failed to connect to Kubernetes cluster: %v", err)
+	}
+	config, err := client.GetSystemConfig()
+	if err != nil {
+		log.Fatalf("Failed to get system config: %v", err)
+	}
+
+	databaseDataSourceName := fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=disable",
+		config["databaseHost"], config["databaseUsername"], config["databasePassword"], config["databaseName"])
+
+	db := sqlx.MustConnect(config["databaseDriverName"], databaseDataSourceName)
+
+	command := args[0]
+
+	arguments := []string{}
+	if len(args) > 2 {
+		arguments = append(arguments, args[2:]...)
+	}
+
+	if err := goose.Run(command, db.DB, *dir, arguments...); err != nil {
+		log.Fatalf("goose %v: %v", command, err)
+	}
+}

db/20200525160514_add_jupyter_workspace_template.go

@@ -0,0 +1,124 @@
+// Package migration is for carrying out migrations against the database.
+// To support Onepanel Core operations.
+package migration
+
+import (
+	"database/sql"
+	v1 "github.com/onepanelio/core/pkg"
+	uid2 "github.com/onepanelio/core/pkg/util/uid"
+	"github.com/pressly/goose"
+	"log"
+)
+
+const jupyterWorkspaceTemplate = `# Docker containers that are part of the Workspace
+containers:
+- name: jupyterlab-tensorflow
+  image: jupyter/tensorflow-notebook
+  command: [start.sh, jupyter]
+  env:
+    - name: tornado
+      value: "{ 'headers': { 'Content-Security-Policy': \"frame-ancestors * 'self'\" }  }"
+  args:
+    - lab
+    - --LabApp.token=''
+    - --LabApp.allow_remote_access=True
+    - --LabApp.allow_origin="*"
+    - --LabApp.disable_check_xsrf=True
+    - --LabApp.trust_xheaders=True
+    - --LabApp.tornado_settings=$(tornado)
+    - --notebook-dir='/data'
+  ports:
+  - containerPort: 8888
+    name: jupyterlab
+  # Volumes to be mounted in this container
+  # Onepanel will automatically create these volumes and mount them to the container
+  volumeMounts:
+  - name: data
+    mountPath: /data
+# Ports that need to be exposed
+ports:
+- name: jupyterlab
+  port: 80
+  protocol: TCP
+  targetPort: 8888
+# Routes that will map to ports
+routes:
+- match:
+  - uri:
+      prefix: /
+  route:
+  - destination:
+      port:
+        number: 80
+# DAG Workflow to be executed once a Workspace action completes
+# postExecutionWorkflow:
+#   entrypoint: main
+#   templates:
+#   - name: main
+#     dag:
+#        tasks:
+#        - name: slack-notify
+#          template: slack-notify
+#   - name: slack-notify
+#     container:
+#       image: technosophos/slack-notify
+#       args:
+#       - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
+#       command:
+#       - sh
+#       - -c
+`
+
+const jupyterLabTemplateName = "JupyterLab"
+
+func init() {
+	goose.AddMigration(Up20200525160514, Down20200525160514)
+}
+
+func Up20200525160514(tx *sql.Tx) error {
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	workspaceTemplate := &v1.WorkspaceTemplate{
+		Name:     jupyterLabTemplateName,
+		Manifest: jupyterWorkspaceTemplate,
+	}
+
+	for _, namespace := range namespaces {
+		if _, err := client.CreateWorkspaceTemplate(namespace.Name, workspaceTemplate); err != nil {
+			log.Fatalf("error %v", err.Error())
+		}
+	}
+
+	return nil
+}
+
+func Down20200525160514(tx *sql.Tx) error {
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+	uid, err := uid2.GenerateUID(jupyterLabTemplateName, 30)
+	if err != nil {
+		return err
+	}
+	for _, namespace := range namespaces {
+		if _, err := client.ArchiveWorkspaceTemplate(namespace.Name, uid); err != nil {
+			log.Fatalf("error %v", err.Error())
+		}
+	}
+
+	return nil
+}

db/20200528140124_add_cvat_workspace_template.go

@@ -0,0 +1,176 @@
+package migration
+
+import (
+	"database/sql"
+	v1 "github.com/onepanelio/core/pkg"
+	uid2 "github.com/onepanelio/core/pkg/util/uid"
+	"github.com/pressly/goose"
+	"log"
+)
+
+const cvatWorkspaceTemplate = `# Docker containers that are part of the Workspace
+containers:
+- name: cvat-db
+  image: postgres:10-alpine
+  env:
+  - name: POSTGRES_USER
+    value: root
+  - name: POSTGRES_DB
+    value: cvat
+  - name: POSTGRES_HOST_AUTH_METHOD
+    value: trust
+  - name: PGDATA
+    value: /var/lib/psql/data
+  ports:
+  - containerPort: 5432
+    name: tcp
+  volumeMounts:
+  - name: db
+    mountPath: /var/lib/psql
+- name: cvat-redis
+  image: redis:4.0-alpine
+  ports:
+  - containerPort: 6379
+    name: tcp
+- name: cvat
+  image: onepanel/cvat:v0.7.0
+  env:
+  - name: DJANGO_MODWSGI_EXTRA_ARGS
+    value: ""
+  - name: ALLOWED_HOSTS
+    value: '*'
+  - name: CVAT_REDIS_HOST
+    value: localhost
+  - name: CVAT_POSTGRES_HOST
+    value: localhost
+  - name: CVAT_SHARE_URL
+    value: /home/django/data
+  ports:
+  - containerPort: 8080
+    name: http
+  volumeMounts:
+  - name: data
+    mountPath: /home/django/data
+  - name: keys
+    mountPath: /home/django/keys
+  - name: logs
+    mountPath: /home/django/logs
+  - name: models
+    mountPath: /home/django/models
+- name: cvat-ui
+  image: onepanel/cvat-ui:v0.7.0
+  ports:
+  - containerPort: 80
+    name: http
+ports:
+- name: cvat-ui
+  port: 80
+  protocol: TCP
+  targetPort: 80
+- name: cvat
+  port: 8080
+  protocol: TCP
+  targetPort: 8080
+routes:
+- match:
+  - uri:
+      regex: /api/.*|/git/.*|/tensorflow/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
+  - queryParams:
+      id:
+        regex: \d+.*
+  route:
+  - destination:
+      port:
+        number: 8080
+- match:
+  - uri:
+      prefix: /
+  route:
+  - destination:
+      port:
+        number: 80
+# DAG Workflow to be executed once a Workspace action completes
+# postExecutionWorkflow:
+#   entrypoint: main
+#   templates:
+#   - name: main
+#     dag:
+#        tasks:
+#        - name: slack-notify
+#          template: slack-notify
+#   - name: slack-notify
+#     container:
+#       image: technosophos/slack-notify
+#       args:
+#       - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
+#       command:
+#       - sh
+#       - -c
+`
+
+const cvatTemplateName = "CVAT"
+
+func init() {
+	goose.AddMigration(Up20200528140124, Down20200528140124)
+}
+
+// Up20200528140124 will insert the cvatTemplate to each user.
+// Each user is determined by onepanel enabled namespaces.
+// Any errors reported are logged as fatal.
+func Up20200528140124(tx *sql.Tx) error {
+	// This code is executed when the migration is applied.
+
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	workspaceTemplate := &v1.WorkspaceTemplate{
+		Name:     cvatTemplateName,
+		Manifest: cvatWorkspaceTemplate,
+	}
+
+	for _, namespace := range namespaces {
+		if _, err := client.CreateWorkspaceTemplate(namespace.Name, workspaceTemplate); err != nil {
+			log.Fatalf("error %v", err.Error())
+		}
+	}
+
+	return nil
+}
+
+// Down20200528140124 will attempt to remove cvatTemplate from each user.
+// Each user is determined by onepanel enabled namespaces.
+// DB entries are archived, K8S components are deleted.
+// Active workspaces with that template are terminated.
+// Any errors reported are logged as fatal.
+func Down20200528140124(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	uid, err := uid2.GenerateUID(cvatTemplateName, 30)
+	if err != nil {
+		return err
+	}
+	for _, namespace := range namespaces {
+		if _, err := client.ArchiveWorkspaceTemplate(namespace.Name, uid); err != nil {
+			log.Fatalf("error %v", err.Error())
+		}
+	}
+
+	return nil
+}

db/20200605090509_add_pytorch_workflow_template.go

@@ -0,0 +1,151 @@
+package migration
+
+import (
+	"database/sql"
+	"log"
+
+	v1 "github.com/onepanelio/core/pkg"
+	uid2 "github.com/onepanelio/core/pkg/util/uid"
+	"github.com/pressly/goose"
+)
+
+const pytorchMnistWorkflowTemplate = `entrypoint: main
+arguments:
+    parameters:
+    - name: source
+      value: https://github.com/onepanelio/pytorch-examples.git
+    - name: command
+      value: "python mnist/main.py --epochs=1"
+volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 2Gi
+  - metadata:
+      name: output
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 2Gi
+templates:
+  - name: main
+    dag:
+      tasks:
+      - name: train-model
+        template: pytorch
+# Uncomment section below to send metrics to Slack
+#      - name: notify-in-slack
+#        dependencies: [train-model]
+#        template: slack-notify-success
+#        arguments:
+#          parameters:
+#          - name: status
+#            value: "{{tasks.train-model.status}}"
+#          artifacts:
+#          - name: metrics
+#            from: "{{tasks.train-model.outputs.artifacts.sys-metrics}}"
+  - name: pytorch
+    inputs:
+      artifacts:
+      - name: src
+        path: /mnt/src
+        git:
+          repo: "{{workflow.parameters.source}}"
+    outputs:
+      artifacts:
+      - name: model
+        path: /mnt/output
+        optional: true
+        archive:
+          none: {}
+    container:
+      image: pytorch/pytorch:latest
+      command: [sh,-c]
+      args: ["{{workflow.parameters.command}}"]
+      workingDir: /mnt/src
+      volumeMounts:
+      - name: data
+        mountPath: /mnt/data
+      - name: output
+        mountPath: /mnt/output
+  - name: slack-notify-success
+    container:
+      image: technosophos/slack-notify
+      command: [sh,-c]
+      args: ['SLACK_USERNAME=Worker SLACK_TITLE="{{workflow.name}} {{inputs.parameters.status}}" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE=$(cat /tmp/metrics.json)} ./slack-notify']
+    inputs:
+      parameters:
+      - name: status
+      artifacts:
+      - name: metrics
+        path: /tmp/metrics.json
+        optional: true
+`
+
+const pytorchMnistWorkflowTemplateName = "PyTorch Training"
+
+func init() {
+	goose.AddMigration(Up20200605090509, Down20200605090509)
+}
+
+// Up20200605090509 will insert a Pytorch workflow template to each user.
+// Each user is determined by onepanel enabled namespaces.
+// Any errors reported are logged as fatal.
+func Up20200605090509(tx *sql.Tx) error {
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	workflowTemplate := &v1.WorkflowTemplate{
+		Name:     pytorchMnistWorkflowTemplateName,
+		Manifest: pytorchMnistWorkflowTemplate,
+	}
+
+	for _, namespace := range namespaces {
+		if _, err := client.CreateWorkflowTemplate(namespace.Name, workflowTemplate); err != nil {
+			log.Fatalf("error %v", err.Error())
+		}
+	}
+
+	return nil
+}
+
+// Down20200605090509 will attempt to remove Pytorch workflow from each user.
+// Each user is determined by onepanel enabled namespaces.
+// DB entries are archived, K8S components are deleted.
+// Active workflows with that template are terminated.
+// Any errors reported are logged as fatal.
+func Down20200605090509(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	uid, err := uid2.GenerateUID(pytorchMnistWorkflowTemplateName, 30)
+	if err != nil {
+		return err
+	}
+	for _, namespace := range namespaces {
+		if _, err := client.ArchiveWorkflowTemplate(namespace.Name, uid); err != nil {
+			log.Fatalf("error %v", err.Error())
+		}
+	}
+
+	return nil
+}

db/20200605090535_add_tensorflow_workflow_template.go

@@ -0,0 +1,150 @@
+package migration
+
+import (
+	"database/sql"
+	"log"
+
+	v1 "github.com/onepanelio/core/pkg"
+	uid2 "github.com/onepanelio/core/pkg/util/uid"
+	"github.com/pressly/goose"
+)
+
+const tensorflowWorkflowTemplate = `entrypoint: main
+arguments:
+    parameters:
+    - name: source
+      value: https://github.com/onepanelio/tensorflow-examples.git
+    - name: command
+      value: "python mnist/main.py --epochs=5"
+volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 2Gi
+  - metadata:
+      name: output
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 2Gi
+templates:
+  - name: main
+    dag:
+      tasks:
+      - name: train-model
+        template: pytorch
+# Uncomment section below to send metrics to Slack
+#      - name: notify-in-slack
+#        dependencies: [train-model]
+#        template: slack-notify-success
+#        arguments:
+#          parameters:
+#          - name: status
+#            value: "{{tasks.train-model.status}}"
+#          artifacts:
+#          - name: metrics
+#            from: "{{tasks.train-model.outputs.artifacts.sys-metrics}}"
+  - name: pytorch
+    inputs:
+      artifacts:
+      - name: src
+        path: /mnt/src
+        git:
+          repo: "{{workflow.parameters.source}}"
+    outputs:
+      artifacts:
+      - name: model
+        path: /mnt/output
+        optional: true
+        archive:
+          none: {}
+    container:
+      image: tensorflow/tensorflow:latest
+      command: [sh,-c]
+      args: ["{{workflow.parameters.command}}"]
+      workingDir: /mnt/src
+      volumeMounts:
+      - name: data
+        mountPath: /mnt/data
+      - name: output
+        mountPath: /mnt/output
+  - name: slack-notify-success
+    container:
+      image: technosophos/slack-notify
+      command: [sh,-c]
+      args: ['SLACK_USERNAME=Worker SLACK_TITLE="{{workflow.name}} {{inputs.parameters.status}}" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE=$(cat /tmp/metrics.json)} ./slack-notify']
+    inputs:
+      parameters:
+      - name: status
+      artifacts:
+      - name: metrics
+        path: /tmp/metrics.json
+        optional: true
+`
+
+const tensorflowWorkflowTemplateName = "TensorFlow Training"
+
+func init() {
+	goose.AddMigration(Up20200605090535, Down20200605090535)
+}
+
+// Up20200605090535 will insert a tensorflow workflow template to each user.
+// Each user is determined by onepanel enabled namespaces.
+// Any errors reported are logged as fatal.
+func Up20200605090535(tx *sql.Tx) error {
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	workflowTemplate := &v1.WorkflowTemplate{
+		Name:     tensorflowWorkflowTemplateName,
+		Manifest: tensorflowWorkflowTemplate,
+	}
+
+	for _, namespace := range namespaces {
+		if _, err := client.CreateWorkflowTemplate(namespace.Name, workflowTemplate); err != nil {
+			log.Fatalf("error %v", err.Error())
+		}
+	}
+	return nil
+}
+
+// Down20200605090535 will attempt to remove tensorflow workflow from each user.
+// Each user is determined by onepanel enabled namespaces.
+// DB entries are archived, K8S components are deleted.
+// Active workflows with that template are terminated.
+// Any errors reported are logged as fatal.
+func Down20200605090535(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	uid, err := uid2.GenerateUID(tensorflowWorkflowTemplateName, 30)
+	if err != nil {
+		return err
+	}
+	for _, namespace := range namespaces {
+		if _, err := client.ArchiveWorkflowTemplate(namespace.Name, uid); err != nil {
+			log.Fatalf("error %v", err.Error())
+		}
+	}
+
+	return nil
+}

db/db.go

@@ -0,0 +1,25 @@
+package migration
+
+import (
+	"fmt"
+	"github.com/jmoiron/sqlx"
+	v1 "github.com/onepanelio/core/pkg"
+)
+
+func getClient() (*v1.Client, error) {
+	kubeConfig := v1.NewConfig()
+	client, err := v1.NewClient(kubeConfig, nil)
+	if err != nil {
+		return nil, err
+	}
+	config, err := client.GetSystemConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	databaseDataSourceName := fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=disable",
+		config["databaseHost"], config["databaseUsername"], config["databasePassword"], config["databaseName"])
+	client.DB = sqlx.MustConnect(config["databaseDriverName"], databaseDataSourceName)
+
+	return client, nil
+}

main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	_ "github.com/onepanelio/core/db"
 	"net"
 	"net/http"
 

pkg/config_types.go

@@ -1,6 +1,9 @@
 package v1
 
-import corev1 "k8s.io/api/core/v1"
+import (
+	corev1 "k8s.io/api/core/v1"
+	"strings"
+)
 
 type ArtifactRepositoryS3Config struct {
 	KeyFormat       string
@@ -14,6 +17,20 @@ type ArtifactRepositoryS3Config struct {
 	Secretkey       string
 }
 
+// FormatKey replaces placeholder values with their actual values and returns this string.
+// {{workflow.namespace}} -> namespace
+// {{workflow.name}} -> workflowName
+// {{pod.name}} -> podName
+func (a *ArtifactRepositoryS3Config) FormatKey(namespace, workflowName, podName string) string {
+	keyFormat := a.KeyFormat
+
+	keyFormat = strings.Replace(keyFormat, "{{workflow.namespace}}", namespace, -1)
+	keyFormat = strings.Replace(keyFormat, "{{workflow.name}}", workflowName, -1)
+	keyFormat = strings.Replace(keyFormat, "{{pod.name}}", podName, -1)
+
+	return keyFormat
+}
+
 type ArtifactRepositoryConfig struct {
 	S3 *ArtifactRepositoryS3Config
 }

pkg/labels.go

@@ -31,6 +31,14 @@ func (c *Client) ListLabels(resource string, uid string) (labels []*Label, err e
 	case TypeCronWorkflow:
 		sb = sb.Join("cron_workflows cw ON cw.id = l.resource_id").
 			Where(sq.Eq{"cw.uid": uid})
+	case TypeWorkspace:
+		sb = sb.Join("workspaces ws ON ws.id = l.resource_id").
+			Where(sq.And{
+				sq.Eq{"ws.uid": uid},
+				sq.NotEq{"ws.phase": "Terminated"},
+			})
+	default:
+		return nil, fmt.Errorf("unsupported label resource %v", resource)
 	}
 
 	query, args, sqlErr := sb.ToSql()
@@ -119,12 +127,24 @@ func (c *Client) ReplaceLabels(namespace, resource, uid string, keyValues map[st
 		return fmt.Errorf("unknown resources '%v'", resource)
 	}
 
+	var whereCondition interface{} = nil
+	if resource == TypeWorkspace {
+		whereCondition = sq.And{
+			sq.Eq{"uid": uid},
+			sq.NotEq{"phase": "Terminated"},
+		}
+	} else if resource == TypeWorkspaceTemplate || resource == TypeWorkflowExecution {
+		whereCondition =
+			sq.Eq{
+				"uid":         uid,
+				"is_archived": false,
+			}
+	}
+
 	resourceID := uint64(0)
 	err = sb.Select("id").
 		From(tableName).
-		Where(sq.Eq{
-			"uid": uid,
-		}).
+		Where(whereCondition).
 		RunWith(tx).
 		QueryRow().
 		Scan(&resourceID)

pkg/types.go

@@ -619,6 +619,21 @@ func CronWorkflowsToIds(resources []*CronWorkflow) (ids []uint64) {
 	return
 }
 
+func WorkspacesToIds(resources []*Workspace) (ids []uint64) {
+	mappedIds := make(map[uint64]bool)
+
+	// This is to make sure we don't have duplicates
+	for _, resource := range resources {
+		mappedIds[resource.ID] = true
+	}
+
+	for id := range mappedIds {
+		ids = append(ids, id)
+	}
+
+	return
+}
+
 // Returns a list of column names prefixed with alias, and named to destination. Extra columns are added to the end of the list.
 // Setting destination to empty string will not apply any destination.
 // Example - with destination

pkg/workflow_execution.go

@@ -825,7 +825,8 @@ func (c *Client) GetWorkflowExecutionLogs(namespace, uid, podName, containerName
 		}
 		opts.SetRange(0, int64(endOffset))
 
-		stream, err = s3Client.GetObject(config.ArtifactRepository.S3.Bucket, "artifacts/"+namespace+"/"+uid+"/"+podName+"/"+containerName+".log", opts)
+		key := config.ArtifactRepository.S3.FormatKey(namespace, uid, podName) + "/" + containerName + ".log"
+		stream, err = s3Client.GetObject(config.ArtifactRepository.S3.Bucket, key, opts)
 	} else {
 		stream, err = c.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{
 			Container:  containerName,
@@ -905,7 +906,8 @@ func (c *Client) GetWorkflowExecutionMetrics(namespace, uid, podName string) (me
 
 	opts := s3.GetObjectOptions{}
 
-	stream, err = s3Client.GetObject(config.ArtifactRepository.S3.Bucket, "artifacts/"+namespace+"/"+uid+"/"+podName+"/sys-metrics.json", opts)
+	key := config.ArtifactRepository.S3.FormatKey(namespace, uid, podName) + "/sys-metrics.json"
+	stream, err = s3Client.GetObject(config.ArtifactRepository.S3.Bucket, key, opts)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"Namespace": namespace,
@@ -1406,9 +1408,6 @@ func injectExitHandlerWorkflowExecutionStatistic(wf *wfv1.Workflow, workflowTemp
 	if wf.Spec.OnExit != "" {
 		for _, t := range wf.Spec.Templates {
 			if t.Name == wf.Spec.OnExit {
-				lasTaskIndex := len(t.DAG.Tasks) - 1
-				dagTask.Dependencies = []string{t.DAG.Tasks[lasTaskIndex].Name}
-
 				t.DAG.Tasks = append(t.DAG.Tasks, dagTask)
 
 				break
@@ -1528,7 +1527,7 @@ func workflowExecutionsSelectBuilderNoColumns(namespace, workflowTemplateUID, wo
 func workflowExecutionsSelectBuilder(namespace, workflowTemplateUID, workflowTemplateVersion string) sq.SelectBuilder {
 	sb := workflowExecutionsSelectBuilderNoColumns(namespace, workflowTemplateUID, workflowTemplateVersion)
 	sb = sb.Columns(getWorkflowExecutionColumns("we", "")...).
-		Columns(`wtv.version "workflow_template.version"`)
+		Columns(`wtv.version "workflow_template.version"`, `wtv.created_at "workflow_template.created_at"`)
 
 	return sb
 }

pkg/workflow_template.go

@@ -198,8 +198,10 @@ func (c *Client) getWorkflowTemplate(namespace, uid string, version int64) (work
 		WorkflowExecutionStatisticReport: &WorkflowExecutionStatisticReport{},
 	}
 
+	// A new workflow template version is created upon a change, so we use it's createdAt
+	// as a modified_at for the workflow template.
 	sb := c.workflowTemplatesSelectBuilder(namespace).
-		Columns("wtv.manifest", "wtv.version", "wtv.id workflow_template_version_id").
+		Columns("wtv.manifest", "wtv.version", "wtv.id workflow_template_version_id", "wtv.created_at modified_at").
 		Join("workflow_template_versions wtv ON wt.id = wtv.workflow_template_id").
 		Where(sq.Eq{
 			"wt.uid":         uid,
@@ -339,6 +341,7 @@ func (c *Client) CountWorkflowTemplates(namespace string) (count int, err error)
 		Where(sq.Eq{
 			"wt.namespace":   namespace,
 			"wt.is_archived": false,
+			"wt.is_system":   false,
 		}).
 		RunWith(c.DB.DB).
 		QueryRow().
@@ -539,6 +542,25 @@ func (c *Client) GetWorkflowTemplateByName(namespace, name string, version int64
 	return
 }
 
+// CountWorkflowTemplateVersions returns the number of versions a non-archived WorkflowTemplate has.
+func (c *Client) CountWorkflowTemplateVersions(namespace, uid string) (count uint64, err error) {
+	count = 0
+
+	err = sb.Select("COUNT(*)").
+		From("workflow_templates wt").
+		Join("workflow_template_versions wtv ON wtv.workflow_template_id = wt.id").
+		Where(sq.Eq{
+			"wt.namespace":   namespace,
+			"wt.uid":         uid,
+			"wt.is_archived": false,
+		}).
+		RunWith(c.DB).
+		QueryRow().
+		Scan(&count)
+
+	return
+}
+
 func (c *Client) ListWorkflowTemplateVersions(namespace, uid string) (workflowTemplateVersions []*WorkflowTemplate, err error) {
 	workflowTemplateVersions, err = c.listWorkflowTemplateVersions(namespace, uid)
 	if err != nil {
@@ -614,9 +636,16 @@ func (c *Client) ArchiveWorkflowTemplate(namespace, uid string) (archived bool,
 	if workflowTemplate == nil {
 		return false, util.NewUserError(codes.NotFound, "Workflow template not found.")
 	}
-	wfTempVer := strconv.FormatInt(workflowTemplate.Version, 10)
-	workflowTemplateName := uid + "-v" + wfTempVer
 
+	wftVersions, err := c.listWorkflowTemplateVersions(namespace, uid)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"Namespace": namespace,
+			"UID":       uid,
+			"Error":     err.Error(),
+		}).Error("Get Workflow Template Versions failed.")
+		return false, util.NewUserError(codes.Unknown, "Unable to archive workflow template.")
+	}
 	//cron workflows
 	cronWorkflows := []*CronWorkflow{}
 	cwfSB := c.cronWorkflowSelectBuilder(namespace, uid).
@@ -654,29 +683,43 @@ func (c *Client) ArchiveWorkflowTemplate(namespace, uid string) (archived bool,
 
 	//workflow executions
 	paginator := pagination.NewRequest(0, 100)
-	for {
-		wfs, err := c.ListWorkflowExecutions(namespace, uid, wfTempVer, &paginator)
-		if err != nil {
-			log.WithFields(log.Fields{
-				"Namespace": namespace,
-				"UID":       uid,
-				"Error":     err.Error(),
-			}).Error("Get Workflow Executions failed.")
-			return false, util.NewUserError(codes.Unknown, "Unable to archive workflow template.")
-		}
-		if len(wfs) == 0 {
-			break
-		}
-		for _, wf := range wfs {
-			err = c.ArchiveWorkflowExecution(namespace, wf.UID)
+
+	for _, wftVer := range wftVersions {
+		wfTempVer := strconv.FormatInt(wftVer.Version, 10)
+		workflowTemplateName := uid + "-v" + wfTempVer
+
+		for {
+			wfs, err := c.ListWorkflowExecutions(namespace, uid, wfTempVer, &paginator)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"Namespace": namespace,
 					"UID":       uid,
 					"Error":     err.Error(),
-				}).Error("Archive Workflow Execution Failed.")
+				}).Error("Get Workflow Executions failed.")
 				return false, util.NewUserError(codes.Unknown, "Unable to archive workflow template.")
 			}
+			if len(wfs) == 0 {
+				break
+			}
+			for _, wf := range wfs {
+				err = c.ArchiveWorkflowExecution(namespace, wf.UID)
+				if err != nil {
+					log.WithFields(log.Fields{
+						"Namespace": namespace,
+						"UID":       uid,
+						"Error":     err.Error(),
+					}).Error("Archive Workflow Execution Failed.")
+					return false, util.NewUserError(codes.Unknown, "Unable to archive workflow template.")
+				}
+			}
+		}
+
+		err = c.ArgoprojV1alpha1().WorkflowTemplates(namespace).Delete(workflowTemplateName, nil)
+		if err != nil {
+			if strings.Contains(err.Error(), "not found") {
+				return true, nil
+			}
+			return false, err
 		}
 	}
 
@@ -696,13 +739,6 @@ func (c *Client) ArchiveWorkflowTemplate(namespace, uid string) (archived bool,
 		return false, util.NewUserError(codes.Unknown, "Unable to archive workflow template.")
 	}
 
-	err = c.ArgoprojV1alpha1().WorkflowTemplates(namespace).Delete(workflowTemplateName, nil)
-	if err != nil {
-		if strings.Contains(err.Error(), "not found") {
-			return true, nil
-		}
-		return false, err
-	}
 	return true, nil
 }
 

pkg/workspace.go

@@ -266,6 +266,40 @@ func (c *Client) UpdateWorkspaceStatus(namespace, uid string, status *WorkspaceS
 	return
 }
 
+// ListWorkspacesByTemplateID will return all the workspaces for a given workspace template id.
+// Sourced from database.
+func (c *Client) ListWorkspacesByTemplateID(namespace string, templateID uint64) (workspaces []*Workspace, err error) {
+	sb := sb.Select(getWorkspaceColumns("w", "")...).
+		From("workspaces w").
+		Where(sq.And{
+			sq.Eq{
+				"w.namespace":             namespace,
+				"w.workspace_template_id": templateID,
+			},
+			sq.NotEq{
+				"phase": WorkspaceTerminated,
+			},
+		})
+	query, args, err := sb.ToSql()
+	if err != nil {
+		return nil, err
+	}
+
+	if err := c.DB.Select(&workspaces, query, args...); err != nil {
+		return nil, err
+	}
+
+	labelMap, err := c.GetDbLabelsMapped(TypeWorkspace, WorkspacesToIds(workspaces)...)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, workspace := range workspaces {
+		workspace.Labels = labelMap[workspace.ID]
+	}
+	return
+}
+
 func (c *Client) ListWorkspaces(namespace string, paginator *pagination.PaginationRequest) (workspaces []*Workspace, err error) {
 	sb := sb.Select(getWorkspaceColumns("w", "")...).
 		Columns(getWorkspaceStatusColumns("w", "status")...).
@@ -292,6 +326,15 @@ func (c *Client) ListWorkspaces(namespace string, paginator *pagination.Paginati
 		return nil, err
 	}
 
+	labelMap, err := c.GetDbLabelsMapped(TypeWorkspace, WorkspacesToIds(workspaces)...)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, workspace := range workspaces {
+		workspace.Labels = labelMap[workspace.ID]
+	}
+
 	return
 }
 
@@ -319,7 +362,9 @@ func (c *Client) updateWorkspace(namespace, uid, workspaceAction, resourceAction
 	if err != nil {
 		return util.NewUserError(codes.NotFound, "Workspace not found.")
 	}
-
+	if workspace == nil {
+		return nil
+	}
 	config, err := c.GetSystemConfig()
 	if err != nil {
 		return
@@ -398,3 +443,9 @@ func (c *Client) ResumeWorkspace(namespace, uid string) (err error) {
 func (c *Client) DeleteWorkspace(namespace, uid string) (err error) {
 	return c.updateWorkspace(namespace, uid, "delete", "delete", &WorkspaceStatus{Phase: WorkspaceTerminating})
 }
+
+// ArchiveWorkspace archives by setting the workspace to delete or terminate.
+// Kicks off DB archiving and k8s cleaning.
+func (c *Client) ArchiveWorkspace(namespace, uid string) (err error) {
+	return c.updateWorkspace(namespace, uid, "delete", "delete", &WorkspaceStatus{Phase: WorkspaceTerminating})
+}

pkg/workspace_template.go

@@ -28,14 +28,9 @@ func parseWorkspaceSpec(template string) (spec *WorkspaceSpec, err error) {
 }
 
 func generateArguments(spec *WorkspaceSpec, config map[string]string) (err error) {
-	if spec.Arguments == nil {
-		spec.Arguments = &Arguments{
-			Parameters: []Parameter{},
-		}
-	}
-
+	systemParameters := make([]Parameter, 0)
 	// Resource action parameter
-	spec.Arguments.Parameters = append(spec.Arguments.Parameters, Parameter{
+	systemParameters = append(systemParameters, Parameter{
 		Name:        "sys-name",
 		Type:        "input.text",
 		Value:       ptr.String("name"),
@@ -46,25 +41,25 @@ func generateArguments(spec *WorkspaceSpec, config map[string]string) (err error
 
 	// TODO: These can be removed when lint validation of workflows work
 	// Resource action parameter
-	spec.Arguments.Parameters = append(spec.Arguments.Parameters, Parameter{
+	systemParameters = append(systemParameters, Parameter{
 		Name:  "sys-resource-action",
 		Value: ptr.String("apply"),
 		Type:  "input.hidden",
 	})
 	// Workspace action
-	spec.Arguments.Parameters = append(spec.Arguments.Parameters, Parameter{
+	systemParameters = append(systemParameters, Parameter{
 		Name:  "sys-workspace-action",
 		Value: ptr.String("create"),
 		Type:  "input.hidden",
 	})
 	// Host
-	spec.Arguments.Parameters = append(spec.Arguments.Parameters, Parameter{
+	systemParameters = append(systemParameters, Parameter{
 		Name:  "sys-host",
 		Value: ptr.String(config["ONEPANEL_DOMAIN"]),
 		Type:  "input.hidden",
 	})
 	// UID placeholder
-	spec.Arguments.Parameters = append(spec.Arguments.Parameters, Parameter{
+	systemParameters = append(systemParameters, Parameter{
 		Name:  "sys-uid",
 		Value: ptr.String("uid"),
 		Type:  "input.hidden",
@@ -75,7 +70,7 @@ func generateArguments(spec *WorkspaceSpec, config map[string]string) (err error
 	if err = yaml.Unmarshal([]byte(config["applicationNodePoolOptions"]), &options); err != nil {
 		return
 	}
-	spec.Arguments.Parameters = append(spec.Arguments.Parameters, Parameter{
+	systemParameters = append(systemParameters, Parameter{
 		Name:        "sys-node-pool",
 		Value:       ptr.String(options[0].Value),
 		Type:        "select.select",
@@ -93,7 +88,7 @@ func generateArguments(spec *WorkspaceSpec, config map[string]string) (err error
 				continue
 			}
 
-			spec.Arguments.Parameters = append(spec.Arguments.Parameters, Parameter{
+			systemParameters = append(systemParameters, Parameter{
 				Name:        fmt.Sprintf("sys-%v-volume-size", v.Name),
 				Type:        "input.number",
 				Value:       ptr.String("20480"),
@@ -106,6 +101,13 @@ func generateArguments(spec *WorkspaceSpec, config map[string]string) (err error
 		}
 	}
 
+	if spec.Arguments == nil {
+		spec.Arguments = &Arguments{
+			Parameters: []Parameter{},
+		}
+	}
+	spec.Arguments.Parameters = append(systemParameters, spec.Arguments.Parameters...)
+
 	return
 }
 
@@ -254,6 +256,11 @@ func unmarshalWorkflowTemplate(spec *WorkspaceSpec, serviceManifest, virtualServ
 		}
 	}
 
+	getStatefulSetManifest := `apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{workflow.parameters.sys-uid}}
+`
 	deletePVCManifest := `apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -263,6 +270,7 @@ metadata:
 		{
 			Name: "workspace",
 			DAG: &wfv1.DAGTemplate{
+				FailFast: ptr.Bool(false),
 				Tasks: []wfv1.DAGTask{
 					{
 						Name:     "service",
@@ -279,6 +287,20 @@ metadata:
 						Dependencies: []string{"virtual-service"},
 						When:         "{{workflow.parameters.sys-workspace-action}} == create || {{workflow.parameters.sys-workspace-action}} == update",
 					},
+					{
+						Name:         "get-stateful-set",
+						Template:     "get-stateful-set-resource",
+						Dependencies: []string{"stateful-set"},
+						When:         "{{workflow.parameters.sys-workspace-action}} == create || {{workflow.parameters.sys-workspace-action}} == update",
+						Arguments: wfv1.Arguments{
+							Parameters: []wfv1.Parameter{
+								{
+									Name:  "update-revision",
+									Value: ptr.String("{{tasks.stateful-set.outputs.parameters.update-revision}}"),
+								},
+							},
+						},
+					},
 					{
 						Name:         "delete-stateful-set",
 						Template:     "delete-stateful-set-resource",
@@ -303,7 +325,7 @@ metadata:
 					{
 						Name:         "sys-set-phase-running",
 						Template:     "sys-update-status",
-						Dependencies: []string{"stateful-set"},
+						Dependencies: []string{"get-stateful-set"},
 						Arguments: wfv1.Arguments{
 							Parameters: []wfv1.Parameter{
 								{
@@ -342,11 +364,6 @@ metadata:
 						},
 						When: "{{workflow.parameters.sys-workspace-action}} == delete",
 					},
-					{
-						Name:         spec.PostExecutionWorkflow.Entrypoint,
-						Template:     spec.PostExecutionWorkflow.Entrypoint,
-						Dependencies: []string{"stateful-set", "delete-stateful-set"},
-					},
 				},
 			},
 		},
@@ -371,6 +388,27 @@ metadata:
 				Manifest:         containersManifest,
 				SuccessCondition: "status.readyReplicas > 0",
 			},
+			Outputs: wfv1.Outputs{
+				Parameters: []wfv1.Parameter{
+					{
+						Name: "update-revision",
+						ValueFrom: &wfv1.ValueFrom{
+							JSONPath: "{.status.updateRevision}",
+						},
+					},
+				},
+			},
+		},
+		{
+			Name: "get-stateful-set-resource",
+			Inputs: wfv1.Inputs{
+				Parameters: []wfv1.Parameter{{Name: "update-revision"}},
+			},
+			Resource: &wfv1.ResourceTemplate{
+				Action:           "get",
+				Manifest:         getStatefulSetManifest,
+				SuccessCondition: "status.readyReplicas > 0, status.currentRevision == {{inputs.parameters.update-revision}}",
+			},
 		},
 		{
 			Name: "delete-stateful-set-resource",
@@ -411,6 +449,14 @@ metadata:
 	templates = append(templates, *curlNodeTemplate)
 	// Add postExecutionWorkflow if it exists
 	if spec.PostExecutionWorkflow != nil {
+		dag := wfv1.DAGTask{
+			Name:         spec.PostExecutionWorkflow.Entrypoint,
+			Template:     spec.PostExecutionWorkflow.Entrypoint,
+			Dependencies: []string{"sys-set-phase-running", "sys-set-phase-paused", "sys-set-phase-terminated"},
+		}
+
+		templates[0].DAG.Tasks = append(templates[0].DAG.Tasks, dag)
+
 		templates = append(templates, spec.PostExecutionWorkflow.Templates...)
 	}
 
@@ -463,8 +509,13 @@ func (c *Client) createWorkspaceTemplate(namespace string, workspaceTemplate *Wo
 		RunWith(tx).
 		QueryRow().Scan(&workspaceTemplate.ID, &workspaceTemplate.CreatedAt)
 	if err != nil {
-		_, err := c.ArchiveWorkflowTemplate(namespace, workspaceTemplate.WorkflowTemplate.UID)
-		return nil, util.NewUserErrorWrap(err, "Workspace template")
+		_, errCleanUp := c.ArchiveWorkflowTemplate(namespace, workspaceTemplate.WorkflowTemplate.UID)
+		errorMsg := "Error with insert into workspace_templates. "
+		if errCleanUp != nil {
+			errorMsg += "Error with clean-up: ArchiveWorkflowTemplate. "
+			errorMsg += errCleanUp.Error()
+		}
+		return nil, util.NewUserErrorWrap(err, errorMsg) //return the source error
 	}
 
 	workspaceTemplateVersionID := uint64(0)
@@ -480,8 +531,13 @@ func (c *Client) createWorkspaceTemplate(namespace string, workspaceTemplate *Wo
 		QueryRow().
 		Scan(&workspaceTemplateVersionID)
 	if err != nil {
-		_, err := c.ArchiveWorkflowTemplate(namespace, workspaceTemplate.WorkflowTemplate.UID)
-		return nil, err
+		_, errCleanUp := c.ArchiveWorkflowTemplate(namespace, workspaceTemplate.WorkflowTemplate.UID)
+		errorMsg := "Error with insert into workspace_templates_versions. "
+		if errCleanUp != nil {
+			errorMsg += "Error with clean-up: ArchiveWorkflowTemplate. "
+			errorMsg += errCleanUp.Error()
+		}
+		return nil, util.NewUserErrorWrap(err, errorMsg) //return the source error
 	}
 
 	if len(workspaceTemplate.Labels) != 0 {
@@ -863,15 +919,13 @@ func (c *Client) WorkspaceTemplateHasRunningWorkspaces(namespace string, uid str
 }
 
 // ArchiveWorkspaceTemplate archives and deletes resources associated with the workspace template.
-// If there is an already archived workspace template, it is left intact, only the un-archived one is considered.
-//
-// If there is no workspace template identified by the parameters, an error is returned with code NotFound.
-//
-// No checks are otherwise made to see if this action is valid.
 //
 // In particular, this action
 //
-// * Marks Workspace Template database record as archived.
+// * Code retrieves all un-archived workspace template versions.
+//
+// * Iterates through each version, grabbing all related workspaces.
+//		- Each workspace is archived (k8s cleaned-up, database entry marked archived)
 //
 // * Marks associated Workflow template as archived
 //
@@ -879,31 +933,57 @@ func (c *Client) WorkspaceTemplateHasRunningWorkspaces(namespace string, uid str
 //
 // * Deletes Workflow Executions in k8s
 func (c *Client) ArchiveWorkspaceTemplate(namespace string, uid string) (archived bool, err error) {
-	workspaceTemplate, err := c.GetWorkspaceTemplate(namespace, uid, 0)
-	if err != nil {
-		return false, util.NewUserError(codes.Unknown, "Unable to get workspace template.")
-	}
-	if workspaceTemplate == nil {
-		return false, util.NewUserError(codes.NotFound, "Workspace template not found.")
-	}
-
-	archived, err = c.archiveWorkspaceTemplateDB(namespace, uid)
+	wsTemps, err := c.ListWorkspaceTemplateVersions(namespace, uid)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"Namespace": namespace,
 			"UID":       uid,
 			"Error":     err.Error(),
-		}).Error("Archive Workspace Template failed.")
+		}).Error("ListWorkspaceTemplateVersions failed.")
 		return false, util.NewUserError(codes.Unknown, "Unable to archive workspace template.")
 	}
-	if !archived {
-		return false, nil
-	}
+	for _, wsTemp := range wsTemps {
+		wsList, err := c.ListWorkspacesByTemplateID(namespace, wsTemp.WorkspaceTemplateVersionID)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"Namespace": namespace,
+				"UID":       uid,
+				"Error":     err.Error(),
+			}).Error("ListWorkspacesByTemplateId failed.")
+			return false, util.NewUserError(codes.Unknown, "Unable to archive workspace template.")
+		}
 
-	archived, err = c.ArchiveWorkflowTemplate(namespace, workspaceTemplate.UID)
-	if err != nil {
-		return false, err
-	}
+		for _, ws := range wsList {
+			err = c.ArchiveWorkspace(namespace, ws.UID)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"Namespace": namespace,
+					"UID":       uid,
+					"Error":     err.Error(),
+				}).Error("ArchiveWorkspace failed.")
+				return false, util.NewUserError(codes.Unknown, "Unable to archive workspace template.")
+			}
+		}
 
+		_, err = c.archiveWorkspaceTemplateDB(namespace, wsTemp.UID)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"Namespace": namespace,
+				"UID":       uid,
+				"Error":     err.Error(),
+			}).Error("Archive Workspace Template DB Failed.")
+			return false, util.NewUserError(codes.Unknown, "Unable to archive workspace template.")
+		}
+
+		_, err = c.ArchiveWorkflowTemplate(namespace, wsTemp.UID)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"Namespace": namespace,
+				"UID":       uid,
+				"Error":     err.Error(),
+			}).Error("Archive Workflow Template Failed.")
+			return false, util.NewUserError(codes.Unknown, "Unable to archive workspace template.")
+		}
+	}
 	return true, nil
 }

server/converter/converter.go

@@ -3,6 +3,7 @@ package converter
 import (
 	"github.com/onepanelio/core/api"
 	v1 "github.com/onepanelio/core/pkg"
+	"sort"
 )
 
 func APIKeyValueToLabel(apiKeyValues []*api.KeyValue) map[string]string {
@@ -28,6 +29,10 @@ func MappingToKeyValue(mapping map[string]string) []*api.KeyValue {
 		})
 	}
 
+	sort.Slice(keyValues, func(i, j int) bool {
+		return keyValues[i].Key < keyValues[j].Key
+	})
+
 	return keyValues
 }
 

server/workflow_template_server.go

@@ -31,6 +31,10 @@ func apiWorkflowTemplate(wft *v1.WorkflowTemplate) *api.WorkflowTemplate {
 		Labels:     converter.MappingToKeyValue(wft.Labels),
 	}
 
+	if wft.ModifiedAt != nil {
+		res.ModifiedAt = wft.ModifiedAt.UTC().Format(time.RFC3339)
+	}
+
 	if wft.WorkflowExecutionStatisticReport != nil {
 		res.Stats = &api.WorkflowExecutionStatisticReport{
 			Total:        wft.WorkflowExecutionStatisticReport.Total,
@@ -135,6 +139,12 @@ func (s *WorkflowTemplateServer) GetWorkflowTemplate(ctx context.Context, req *a
 		return nil, err
 	}
 
+	versionsCount, err := client.CountWorkflowTemplateVersions(req.Namespace, req.Uid)
+	if err != nil {
+		return nil, err
+	}
+	workflowTemplate.Versions = int64(versionsCount)
+
 	return apiWorkflowTemplate(workflowTemplate), nil
 }