Merge pull request #675 from Vafilor/feat/cvat.upgrade

feat: upgrade cvat to v14

.github/workflows/push_tag.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Notify Slack Channels
         uses: rtCamp/action-slack-notify@v2.0.0
         env:
-          SLACK_CHANNEL: dev
+          SLACK_CHANNEL: org
           SLACK_ICON: https://avatars1.githubusercontent.com/u/30390575?s=48&v=4
           SLACK_TITLE: New Core Version
           SLACK_USERNAME: opBot

db/data/cvat_20201016170415.yaml

@@ -0,0 +1,147 @@
+# Workspace arguments
+arguments:
+  parameters:
+    - name: sync-directory
+      displayName: Directory to sync raw input and training output
+      value: workflow-data
+      hint: Location to sync raw input, models and checkpoints from default object storage. Note that this will be relative to the current namespace.
+containers:
+  - name: cvat-db
+    image: postgres:10-alpine
+    env:
+      - name: POSTGRES_USER
+        value: root
+      - name: POSTGRES_DB
+        value: cvat
+      - name: POSTGRES_HOST_AUTH_METHOD
+        value: trust
+      - name: PGDATA
+        value: /var/lib/psql/data
+    ports:
+      - containerPort: 5432
+        name: tcp
+    volumeMounts:
+      - name: db
+        mountPath: /var/lib/psql
+  - name: cvat-redis
+    image: redis:4.0-alpine
+    ports:
+      - containerPort: 6379
+        name: tcp
+  - name: cvat
+    image: onepanel/cvat:0.14.0_cvat.1.0.0
+    env:
+      - name: DJANGO_MODWSGI_EXTRA_ARGS
+        value: ""
+      - name: ALLOWED_HOSTS
+        value: '*'
+      - name: CVAT_REDIS_HOST
+        value: localhost
+      - name: CVAT_POSTGRES_HOST
+        value: localhost
+      - name: CVAT_SHARE_URL
+        value: /home/django/data
+      - name: ONEPANEL_SYNC_DIRECTORY
+        value: '{{workspace.parameters.sync-directory}}'
+      - name: NVIDIA_VISIBLE_DEVICES
+        value: all
+      - name: NVIDIA_DRIVER_CAPABILITIES
+        value: compute,utility
+      - name: NVIDIA_REQUIRE_CUDA
+        value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
+    ports:
+      - containerPort: 8080
+        name: http
+    volumeMounts:
+      - name: data
+        mountPath: /home/django/data
+      - name: keys
+        mountPath: /home/django/keys
+      - name: logs
+        mountPath: /home/django/logs
+      - name: models
+        mountPath: /home/django/models
+      - name: share
+        mountPath: /home/django/share
+      - name: sys-namespace-config
+        mountPath: /etc/onepanel
+        readOnly: true
+  - name: cvat-ui
+    image: onepanel/cvat-ui:0.14.0_cvat.1.0.0
+    ports:
+      - containerPort: 80
+        name: http
+  # You can add multiple FileSyncer sidecar containers if needed
+  - name: filesyncer
+    image: onepanel/filesyncer:s3
+    imagePullPolicy: Always
+    args:
+      - download
+      - -server-prefix=/sys/filesyncer
+    env:
+      - name: FS_PATH
+        value: /mnt/share
+      - name: FS_PREFIX
+        value: '{{workflow.namespace}}/{{workspace.parameters.sync-directory}}'
+    volumeMounts:
+      - name: share
+        mountPath: /mnt/share
+      - name: sys-namespace-config
+        mountPath: /etc/onepanel
+        readOnly: true
+ports:
+  - name: cvat-ui
+    port: 80
+    protocol: TCP
+    targetPort: 80
+  - name: cvat
+    port: 8080
+    protocol: TCP
+    targetPort: 8080
+  - name: fs
+    port: 8888
+    protocol: TCP
+    targetPort: 8888
+routes:
+  - match:
+      - uri:
+          prefix: /sys/filesyncer
+    route:
+      - destination:
+          port:
+            number: 8888
+  - match:
+      - uri:
+          regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
+      - queryParams:
+          id:
+            regex: \d+.*
+    route:
+      - destination:
+          port:
+            number: 8080
+  - match:
+      - uri:
+          prefix: /
+    route:
+      - destination:
+          port:
+            number: 80
+# DAG Workflow to be executed once a Workspace action completes (optional)
+# Uncomment the lines below if you want to send Slack notifications
+#postExecutionWorkflow:
+#  entrypoint: main
+#  templates:
+#  - name: main
+#    dag:
+#       tasks:
+#       - name: slack-notify
+#         template: slack-notify
+#  - name: slack-notify
+#     container:
+#       image: technosophos/slack-notify
+#       args:
+#       - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
+#       command:
+#       - sh
+#       - -c

db/go/20201016170415_update_cvat.go

@@ -0,0 +1,25 @@
+package migration
+
+import (
+	"database/sql"
+	"github.com/pressly/goose"
+)
+
+func initialize20201016170415() {
+	if _, ok := initializedMigrations[20201016170415]; !ok {
+		goose.AddMigration(Up20201016170415, Down20201016170415)
+		initializedMigrations[20201016170415] = true
+	}
+}
+
+// Up20201016170415 updates cvat to a new version
+func Up20201016170415(tx *sql.Tx) error {
+	// This code is executed when the migration is applied.
+	return updateWorkspaceTemplateManifest("cvat_20201016170415.yaml", cvatTemplateName)
+}
+
+// Down20201016170415 does nothing
+func Down20201016170415(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	return nil
+}

db/go/db.go

@@ -5,7 +5,10 @@ import (
 	sq "github.com/Masterminds/squirrel"
 	"github.com/jmoiron/sqlx"
 	v1 "github.com/onepanelio/core/pkg"
+	"io/ioutil"
 	"log"
+	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -60,6 +63,7 @@ func Initialize() {
 	initialize20200929144301()
 	initialize20200929153931()
 	initialize20201001070806()
+	initialize20201016170415()
 
 	if err := client.DB.Close(); err != nil {
 		log.Printf("[error] closing db %v", err)
@@ -128,3 +132,17 @@ func ReplaceArtifactRepositoryType(client *v1.Client, namespace *v1.Namespace, w
 
 	return nil
 }
+
+// readDataFile returns the contents of a file in the db/data/{name} directory
+func readDataFile(name string) (string, error) {
+	curDir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	data, err := ioutil.ReadFile(filepath.Join(curDir, "db", "data", name))
+	if err != nil {
+		return "", err
+	}
+
+	return string(data), nil
+}

db/go/util.go

@@ -0,0 +1,49 @@
+package migration
+
+import (
+	v1 "github.com/onepanelio/core/pkg"
+	uid2 "github.com/onepanelio/core/pkg/util/uid"
+)
+
+// updateWorkspaceTemplateManifest will update the workspace template given by {{templateName}} with the contents
+// given by {{filename}}
+// It will do so for all namespaces.
+func updateWorkspaceTemplateManifest(filename, templateName string) error {
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+	defer client.DB.Close()
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	newManifest, err := readDataFile(filename)
+	if err != nil {
+		return err
+	}
+
+	uid, err := uid2.GenerateUID(templateName, 30)
+	if err != nil {
+		return err
+	}
+
+	for _, namespace := range namespaces {
+		workspaceTemplate := &v1.WorkspaceTemplate{
+			UID:      uid,
+			Name:     templateName,
+			Manifest: newManifest,
+		}
+		err = ReplaceArtifactRepositoryType(client, namespace, nil, workspaceTemplate)
+		if err != nil {
+			return err
+		}
+		if _, err := client.UpdateWorkspaceTemplateManifest(namespace.Name, uid, workspaceTemplate.Manifest); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	migrations "github.com/onepanelio/core/db/go"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	corev1 "k8s.io/api/core/v1"
@@ -25,7 +26,6 @@ import (
 	"github.com/grpc-ecosystem/grpc-gateway/runtime"
 	"github.com/jmoiron/sqlx"
 	"github.com/onepanelio/core/api"
-	migrations "github.com/onepanelio/core/db/go"
 	v1 "github.com/onepanelio/core/pkg"
 	"github.com/onepanelio/core/pkg/util/env"
 	"github.com/onepanelio/core/server"

pkg/client.go

@@ -57,6 +57,19 @@ func GetDefaultClient() (*Client, error) {
 	return client, nil
 }
 
+// GetDefaultClientWithDB loads a default k8s client with an existing DB
+func GetDefaultClientWithDB(db *DB) (*Client, error) {
+	kubeConfig := NewConfig()
+	client, err := NewClient(kubeConfig, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	client.DB = db
+
+	return client, nil
+}
+
 // NewClient creates a client to interact with the Onepanel system.
 // It includes access to the database, kubernetes, argo, and configuration.
 func NewClient(config *Config, db *DB, systemConfig SystemConfig) (client *Client, err error) {

pkg/workflow_execution.go

@@ -1340,7 +1340,7 @@ func (c *Client) ListFiles(namespace, key string) (files []*File, err error) {
 
 			doneCh := make(chan struct{})
 			defer close(doneCh)
-			for objInfo := range s3Client.ListObjectsV2(config.ArtifactRepository.S3.Bucket, key, false, doneCh) {
+			for objInfo := range s3Client.ListObjects(config.ArtifactRepository.S3.Bucket, key, false, doneCh) {
 				if objInfo.Key == key {
 					continue
 				}

pkg/workspace_template.go

@@ -401,7 +401,7 @@ func createStatefulSetManifest(spec *WorkspaceSpec, config map[string]string, se
 		env.PrependEnvVarToContainer(container, "ONEPANEL_DOMAIN", config["ONEPANEL_DOMAIN"])
 		env.PrependEnvVarToContainer(container, "ONEPANEL_PROVIDER", config["ONEPANEL_PROVIDER"])
 		env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_NAMESPACE", "{{workflow.namespace}}")
-		env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_UID", "{{workflow.parameters.sys-name}}")
+		env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_UID", "{{workflow.parameters.sys-uid}}")
 
 		for _, service := range services {
 			envName := fmt.Sprintf("ONEPANEL_SERVICES_%v_API_URL", strings.ToUpper(service.Name))

server/auth/auth.go

@@ -165,7 +165,7 @@ func UnaryInterceptor(kubeConfig *v1.Config, db *v1.DB, sysConfig v1.SystemConfi
 				return resp, errors.New("LogInRequest does not have correct request type")
 			}
 
-			defaultClient, err := v1.GetDefaultClient()
+			defaultClient, err := v1.GetDefaultClientWithDB(db)
 			if err != nil {
 				return nil, err
 			}