Merge pull request #678 from rushtehrani/docs/readme

docs: Update README

.github/workflows/push_tag.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Notify Slack Channels
         uses: rtCamp/action-slack-notify@v2.0.0
         env:
-          SLACK_CHANNEL: dev
+          SLACK_CHANNEL: org
           SLACK_ICON: https://avatars1.githubusercontent.com/u/30390575?s=48&v=4
           SLACK_TITLE: New Core Version
           SLACK_USERNAME: opBot

README.md

@@ -9,57 +9,34 @@
 [![chat](https://img.shields.io/badge/support-slack-01579b)](https://onepanel-ce.slack.com/join/shared_invite/zt-eyjnwec0-nLaHhjif9Y~gA05KuX6AUg#/)
 [![license](https://img.shields.io/github/license/onepanelio/core?color=01579b)](https://opensource.org/licenses/Apache-2.0)
 
-Production scale, Kubernetes-native vision AI platform, with fully integrated components for model building, automated labeling, data processing and model training pipelines.
+Production scale, vision AI platform, with fully integrated components for model building, automated labeling, data processing and model training pipelines.
+
+<img width="100%" src="img/onepanel.gif">
 
 ## Why Onepanel?
 
--  End-to-end workflow and infrastructure automation for production scale vision AI
--  Automatic resource management and on-demand scaling of CPU and GPU nodes
--  Easily scale your data processing and training pipelines to multiple nodes
--  Collaborate on all your deep learning tools and workflows through a unified web interface and SDKs
+-  End-to-end automation for production scale vision AI pipelines
+-  Best of breed, open source deep learning tools seamlessly integrated in one unified platform
+-  Infrastructure automation so you can easily scale your data processing and training pipelines to multiple nodes
+-  Customizable, reproducible and version controlled tooling and pipeline templates
 -  Scalability, flexibility and resiliency of Kubernetes without the deployment and configuration complexities
 
 ## Features
-<table>
-  <tr>
-    <td width="50%" align="center">
-      <h3>Image and video annotation with automatic annotation</h3>
-      <img width="100%" src="img/auto-annotation.gif">
-      <p>
-        Annotate images and video with automatic annotation of bounding boxes and polygon masks, integrated with training pipelines to iteratively improve models for pre-annotation and inference
-      </p>
-    </td>
-    <td width="50%" align="center">
-      <h3>JupyterLab with TensorFlow, PyTorch and GPU support</h3>
-      <img width="100%" src="img/jupyterlab.gif">
-      <p>
-        JupyterLab configured with extensions for debugging, Git/GitHub, notebook diffing and TensorBoard and support for Conda, OpenCV, Tensorflow and PyTorch with GPU and <a href="https://github.com/onepanelio/templates/tree/master/workspaces/jupyterlab">much more</a>
-      </p>
-    </td>
-  </tr>
-  <tr>
-    <td width="50%" align="center">
-      <h3>Auto scaling, distributed and parallel data processing and training pipelines</h3>
-      <img width="100%" src="img/pipelines.gif">
-      <p>
-        Build fully reproducible, distributed and parallel data processing and training pipelines with real-time logs and output snapshots
-      </p>
-    </td>
-    <td width="50%" align="center">
-      <h3>Version controlled pipelines and environments as code</h3>
-      <img width="100%" src="img/tools.gif">
-      <p>
-        Bring your own IDEs, annotation tools and pipelines with a version controlled YAML and Docker based template engine
-      </p>
-    </td>
-  </tr>
-</table>
-
+-  Annotate images and video with automatic annotation of bounding boxes and polygon masks, integrated with training pipelines to iteratively improve models for pre-annotation and inference.
+-  JupyterLab configured with extensions for debugging, Git/GitHub, notebook diffing and TensorBoard and support for Conda, OpenCV, Tensorflow and PyTorch with GPU and <a href="https://github.com/onepanelio/templates/tree/master/workspaces/jupyterlab">much more</a>.
+-  Build fully reproducible, distributed and parallel data processing and training pipelines with real-time logs and output snapshots.
+-  Bring your own IDEs, annotation tools and pipelines with a version controlled YAML and Docker based template engine.
 -  Track and visualize model metrics and experiments with TensorBoard or bring your own experiment tracking tools.
--  Access and share tools like AirSim, Carla, Gazebo or OpenAI Gym through your browser with VNC enabled workspaces.
--  Extend Onepanel with powerful REST APIs and SDKs to further automate your pipelines and environments.
--  Workflows, environments and infrastructure are all defined as code and version controlled, making them reproducible and portable.
--  Powered by Kubernetes so you can deploy anywhere Kubernetes can run.
+-  Extend Onepanel with powerful REST APIs and SDKs to further automate your workflows.
+
+## Online demo
+We have created an [online demo environment](https://onepanel.typeform.com/to/kQfDX5Vf?product=github) so that you can quickly try Onepanel.
+
+Note that this is a shared demo environment with the following restrictions:
+
+- Data is reset every few hours
+- One type of node pool (machine type) with a limit of 5 concurrent nodes
+- Certain actions may be restricted
 
 ## Quick start
 See [quick start guide](https://docs.onepanel.ai/docs/getting-started/quickstart) to get started with the platform of your choice.
@@ -74,17 +51,17 @@ See [documentation](https://docs.onepanel.ai) to get started or for more detaile
 
 To submit a feature request, report a bug or documentation issue, please open a GitHub [pull request](https://github.com/onepanelio/core/pulls) or [issue](https://github.com/onepanelio/core/issues).
 
-For help, questions, release announcements and contribution discussions, join us on [Slack](https://join.slack.com/t/onepanel-ce/shared_invite/zt-eyjnwec0-nLaHhjif9Y~gA05KuX6AUg).
+For help, questions, release announcements and contribution discussions, join us on [GitHub discussions](https://github.com/onepanelio/core/discussions).
 
 ## Contributing
 
 Onepanel is modular and consists of the following repositories:
 
-[Core API](https://github.com/onepanelio/core/) (this repository) - Code base for backend (Go)\
-[Core UI](https://github.com/onepanelio/core-ui/) - Code base for UI (Angular + TypeScript)\
-[CLI](https://github.com/onepanelio/cli/) - Code base for Go CLI for installation and management (Go)\
-[Manifests](https://github.com/onepanelio/core-ui/) - Kustomize manifests used by CLI for installation and management (YAML)\
-[Python SDK](https://github.com/onepanelio/python-sdk/) - Python SDK code and documentation\
+[Backend](https://github.com/onepanelio/core/) (this repository) - Code base for backend (Go)\
+[Frontend](https://github.com/onepanelio/core-ui/) - Code base for frontend (Angular + TypeScript)\
+[CLI](https://github.com/onepanelio/cli/) - Code base for installation and management CLI (Go)\
+[Manifests](https://github.com/onepanelio/core-ui/) - Kustomize manifests used by installation and management CLI (YAML)\
+[Python SDK](https://github.com/onepanelio/python-sdk/) - Python SDK code and documentation (Python)\
 [Templates](https://github.com/onepanelio/templates) - Various Workspace, Workflow, Task and Sidecar Templates\
 [Documentation](https://github.com/onepanelio/core-docs/) - The repository for documentation site\
 [API Documentation](https://github.com/onepanelio/core-api-docs/) - API documentation if you choose to use the API directly

api/api.swagger.json

@@ -2984,7 +2984,7 @@
         "domain": {
           "type": "string"
         },
-        "jwtToken": {
+        "token": {
           "type": "string"
         },
         "username": {

api/auth.pb.go

@@ -265,7 +265,7 @@ type IsValidTokenResponse struct {
 	unknownFields protoimpl.UnknownFields
 
 	Domain   string `protobuf:"bytes,1,opt,name=domain,proto3" json:"domain,omitempty"`
-	JwtToken string `protobuf:"bytes,2,opt,name=jwtToken,proto3" json:"jwtToken,omitempty"`
+	Token    string `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"`
 	Username string `protobuf:"bytes,3,opt,name=username,proto3" json:"username,omitempty"`
 }
 
@@ -308,9 +308,9 @@ func (x *IsValidTokenResponse) GetDomain() string {
 	return ""
 }
 
-func (x *IsValidTokenResponse) GetJwtToken() string {
+func (x *IsValidTokenResponse) GetToken() string {
 	if x != nil {
-		return x.JwtToken
+		return x.Token
 	}
 	return ""
 }
@@ -352,28 +352,28 @@ var file_auth_proto_rawDesc = []byte{
 	0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
 	0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14,
 	0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74,
-	0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x66, 0x0a, 0x14, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54,
+	0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x60, 0x0a, 0x14, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54,
 	0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06,
 	0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f,
-	0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x6a, 0x77, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6a, 0x77, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
-	0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x32, 0xe6, 0x01, 0x0a,
-	0x0b, 0x41, 0x75, 0x74, 0x68, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x68, 0x0a, 0x0c,
-	0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x2e, 0x61,
-	0x70, 0x69, 0x2e, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x56,
-	0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x22, 0x23, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1d, 0x22, 0x18, 0x2f, 0x61, 0x70, 0x69, 0x73,
-	0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x61, 0x75, 0x74, 0x68, 0x2f, 0x74, 0x6f,
-	0x6b, 0x65, 0x6e, 0x3a, 0x01, 0x2a, 0x12, 0x6d, 0x0a, 0x0c, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68,
-	0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x41,
-	0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69,
-	0x7a, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x28, 0x82, 0xd3, 0xe4,
-	0x93, 0x02, 0x22, 0x22, 0x12, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74,
-	0x61, 0x31, 0x2f, 0x61, 0x75, 0x74, 0x68, 0x3a, 0x0c, 0x69, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f,
-	0x72, 0x69, 0x7a, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6d, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73,
+	0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73,
+	0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x32, 0xe6, 0x01, 0x0a, 0x0b, 0x41, 0x75, 0x74, 0x68, 0x53,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x68, 0x0a, 0x0c, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69,
+	0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x56,
+	0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f,
+	0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x23, 0x82, 0xd3, 0xe4,
+	0x93, 0x02, 0x1d, 0x22, 0x18, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74,
+	0x61, 0x31, 0x2f, 0x61, 0x75, 0x74, 0x68, 0x2f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x3a, 0x01, 0x2a,
+	0x12, 0x6d, 0x0a, 0x0c, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64,
+	0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69,
+	0x7a, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69,
+	0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x28, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x22, 0x22, 0x12, 0x2f,
+	0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x61, 0x75, 0x74,
+	0x68, 0x3a, 0x0c, 0x69, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x62,
+	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

api/auth.proto

@@ -44,6 +44,6 @@ message IsValidTokenRequest {
 
 message IsValidTokenResponse {
     string domain = 1;
-    string jwtToken = 2;
+    string token = 2;
     string username = 3;
 }

db/data/cvat_20201016170415.yaml

@@ -0,0 +1,147 @@
+# Workspace arguments
+arguments:
+  parameters:
+    - name: sync-directory
+      displayName: Directory to sync raw input and training output
+      value: workflow-data
+      hint: Location to sync raw input, models and checkpoints from default object storage. Note that this will be relative to the current namespace.
+containers:
+  - name: cvat-db
+    image: postgres:10-alpine
+    env:
+      - name: POSTGRES_USER
+        value: root
+      - name: POSTGRES_DB
+        value: cvat
+      - name: POSTGRES_HOST_AUTH_METHOD
+        value: trust
+      - name: PGDATA
+        value: /var/lib/psql/data
+    ports:
+      - containerPort: 5432
+        name: tcp
+    volumeMounts:
+      - name: db
+        mountPath: /var/lib/psql
+  - name: cvat-redis
+    image: redis:4.0-alpine
+    ports:
+      - containerPort: 6379
+        name: tcp
+  - name: cvat
+    image: onepanel/cvat:0.14.0_cvat.1.0.0
+    env:
+      - name: DJANGO_MODWSGI_EXTRA_ARGS
+        value: ""
+      - name: ALLOWED_HOSTS
+        value: '*'
+      - name: CVAT_REDIS_HOST
+        value: localhost
+      - name: CVAT_POSTGRES_HOST
+        value: localhost
+      - name: CVAT_SHARE_URL
+        value: /home/django/data
+      - name: ONEPANEL_SYNC_DIRECTORY
+        value: '{{workspace.parameters.sync-directory}}'
+      - name: NVIDIA_VISIBLE_DEVICES
+        value: all
+      - name: NVIDIA_DRIVER_CAPABILITIES
+        value: compute,utility
+      - name: NVIDIA_REQUIRE_CUDA
+        value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
+    ports:
+      - containerPort: 8080
+        name: http
+    volumeMounts:
+      - name: data
+        mountPath: /home/django/data
+      - name: keys
+        mountPath: /home/django/keys
+      - name: logs
+        mountPath: /home/django/logs
+      - name: models
+        mountPath: /home/django/models
+      - name: share
+        mountPath: /home/django/share
+      - name: sys-namespace-config
+        mountPath: /etc/onepanel
+        readOnly: true
+  - name: cvat-ui
+    image: onepanel/cvat-ui:0.14.0_cvat.1.0.0
+    ports:
+      - containerPort: 80
+        name: http
+  # You can add multiple FileSyncer sidecar containers if needed
+  - name: filesyncer
+    image: onepanel/filesyncer:{{.ArtifactRepositoryType}}
+    imagePullPolicy: Always
+    args:
+      - download
+      - -server-prefix=/sys/filesyncer
+    env:
+      - name: FS_PATH
+        value: /mnt/share
+      - name: FS_PREFIX
+        value: '{{workflow.namespace}}/{{workspace.parameters.sync-directory}}'
+    volumeMounts:
+      - name: share
+        mountPath: /mnt/share
+      - name: sys-namespace-config
+        mountPath: /etc/onepanel
+        readOnly: true
+ports:
+  - name: cvat-ui
+    port: 80
+    protocol: TCP
+    targetPort: 80
+  - name: cvat
+    port: 8080
+    protocol: TCP
+    targetPort: 8080
+  - name: fs
+    port: 8888
+    protocol: TCP
+    targetPort: 8888
+routes:
+  - match:
+      - uri:
+          prefix: /sys/filesyncer
+    route:
+      - destination:
+          port:
+            number: 8888
+  - match:
+      - uri:
+          regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
+      - queryParams:
+          id:
+            regex: \d+.*
+    route:
+      - destination:
+          port:
+            number: 8080
+  - match:
+      - uri:
+          prefix: /
+    route:
+      - destination:
+          port:
+            number: 80
+# DAG Workflow to be executed once a Workspace action completes (optional)
+# Uncomment the lines below if you want to send Slack notifications
+#postExecutionWorkflow:
+#  entrypoint: main
+#  templates:
+#  - name: main
+#    dag:
+#       tasks:
+#       - name: slack-notify
+#         template: slack-notify
+#  - name: slack-notify
+#     container:
+#       image: technosophos/slack-notify
+#       args:
+#       - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
+#       command:
+#       - sh
+#       - -c

db/go/20201016170415_update_cvat.go

@@ -0,0 +1,25 @@
+package migration
+
+import (
+	"database/sql"
+	"github.com/pressly/goose"
+)
+
+func initialize20201016170415() {
+	if _, ok := initializedMigrations[20201016170415]; !ok {
+		goose.AddMigration(Up20201016170415, Down20201016170415)
+		initializedMigrations[20201016170415] = true
+	}
+}
+
+// Up20201016170415 updates cvat to a new version
+func Up20201016170415(tx *sql.Tx) error {
+	// This code is executed when the migration is applied.
+	return updateWorkspaceTemplateManifest("cvat_20201016170415.yaml", cvatTemplateName)
+}
+
+// Down20201016170415 does nothing
+func Down20201016170415(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	return nil
+}

db/go/db.go

@@ -5,7 +5,10 @@ import (
 	sq "github.com/Masterminds/squirrel"
 	"github.com/jmoiron/sqlx"
 	v1 "github.com/onepanelio/core/pkg"
+	"io/ioutil"
 	"log"
+	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -60,6 +63,7 @@ func Initialize() {
 	initialize20200929144301()
 	initialize20200929153931()
 	initialize20201001070806()
+	initialize20201016170415()
 
 	if err := client.DB.Close(); err != nil {
 		log.Printf("[error] closing db %v", err)
@@ -128,3 +132,17 @@ func ReplaceArtifactRepositoryType(client *v1.Client, namespace *v1.Namespace, w
 
 	return nil
 }
+
+// readDataFile returns the contents of a file in the db/data/{name} directory
+func readDataFile(name string) (string, error) {
+	curDir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	data, err := ioutil.ReadFile(filepath.Join(curDir, "db", "data", name))
+	if err != nil {
+		return "", err
+	}
+
+	return string(data), nil
+}

db/go/util.go

@@ -0,0 +1,49 @@
+package migration
+
+import (
+	v1 "github.com/onepanelio/core/pkg"
+	uid2 "github.com/onepanelio/core/pkg/util/uid"
+)
+
+// updateWorkspaceTemplateManifest will update the workspace template given by {{templateName}} with the contents
+// given by {{filename}}
+// It will do so for all namespaces.
+func updateWorkspaceTemplateManifest(filename, templateName string) error {
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+	defer client.DB.Close()
+
+	namespaces, err := client.ListOnepanelEnabledNamespaces()
+	if err != nil {
+		return err
+	}
+
+	newManifest, err := readDataFile(filename)
+	if err != nil {
+		return err
+	}
+
+	uid, err := uid2.GenerateUID(templateName, 30)
+	if err != nil {
+		return err
+	}
+
+	for _, namespace := range namespaces {
+		workspaceTemplate := &v1.WorkspaceTemplate{
+			UID:      uid,
+			Name:     templateName,
+			Manifest: newManifest,
+		}
+		err = ReplaceArtifactRepositoryType(client, namespace, nil, workspaceTemplate)
+		if err != nil {
+			return err
+		}
+		if _, err := client.UpdateWorkspaceTemplateManifest(namespace.Name, uid, workspaceTemplate.Manifest); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	migrations "github.com/onepanelio/core/db/go"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	corev1 "k8s.io/api/core/v1"
@@ -25,7 +26,6 @@ import (
 	"github.com/grpc-ecosystem/grpc-gateway/runtime"
 	"github.com/jmoiron/sqlx"
 	"github.com/onepanelio/core/api"
-	migrations "github.com/onepanelio/core/db/go"
 	v1 "github.com/onepanelio/core/pkg"
 	"github.com/onepanelio/core/pkg/util/env"
 	"github.com/onepanelio/core/server"

pkg/client.go

@@ -57,6 +57,19 @@ func GetDefaultClient() (*Client, error) {
 	return client, nil
 }
 
+// GetDefaultClientWithDB loads a default k8s client with an existing DB
+func GetDefaultClientWithDB(db *DB) (*Client, error) {
+	kubeConfig := NewConfig()
+	client, err := NewClient(kubeConfig, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	client.DB = db
+
+	return client, nil
+}
+
 // NewClient creates a client to interact with the Onepanel system.
 // It includes access to the database, kubernetes, argo, and configuration.
 func NewClient(config *Config, db *DB, systemConfig SystemConfig) (client *Client, err error) {

pkg/util/tokens/jwt.go

@@ -1,48 +0,0 @@
-package tokens
-
-import (
-	"fmt"
-	"github.com/dgrijalva/jwt-go"
-)
-
-// TokenContent represents the content we store in a JWT token - the username and k8s token
-type TokenContent struct {
-	Username string
-	Token    string
-}
-
-// CreateJWTToken creates a jwt token containing a username and another token using the input secret
-func CreateJWTToken(username string, token string, secret []byte) (string, error) {
-	result := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
-		"username": username,
-		"token":    token,
-	})
-
-	// Sign and get the complete encoded token as a string using the secret
-	return result.SignedString(secret)
-}
-
-// ParseJWTToken parses the token string into a TokenContent
-func ParseJWTToken(tokenString string, secret []byte) (content *TokenContent, err error) {
-	token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
-		// Don't forget to validate the alg is what you expect:
-		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
-			return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
-		}
-
-		return secret, nil
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	if claims, ok := token.Claims.(jwt.MapClaims); ok && token.Valid {
-		return &TokenContent{
-			Username: claims["username"].(string),
-			Token:    claims["token"].(string),
-		}, nil
-	}
-
-	return nil, fmt.Errorf("Unknown error getting token, claim or token is not ok")
-}

pkg/workflow_execution.go

@@ -1340,7 +1340,7 @@ func (c *Client) ListFiles(namespace, key string) (files []*File, err error) {
 
 			doneCh := make(chan struct{})
 			defer close(doneCh)
-			for objInfo := range s3Client.ListObjectsV2(config.ArtifactRepository.S3.Bucket, key, false, doneCh) {
+			for objInfo := range s3Client.ListObjects(config.ArtifactRepository.S3.Bucket, key, false, doneCh) {
 				if objInfo.Key == key {
 					continue
 				}
@@ -1851,14 +1851,14 @@ func workflowExecutionsSelectBuilderNoColumns(namespace, workflowTemplateUID, wo
 
 func workflowExecutionsSelectBuilder(namespace, workflowTemplateUID, workflowTemplateVersion string, includeSystem bool) sq.SelectBuilder {
 	sb := workflowExecutionsSelectBuilderNoColumns(namespace, workflowTemplateUID, workflowTemplateVersion, includeSystem)
-	sb = sb.Columns(getWorkflowExecutionColumns("we", "")...).
+	sb = sb.Columns(getWorkflowExecutionColumns("we")...).
 		Columns(`wtv.version "workflow_template.version"`, `wtv.created_at "workflow_template.created_at"`, `wt.name "workflow_template.name"`, `wt.uid "workflow_template.uid"`)
 
 	return sb
 }
 
 func (c *Client) getWorkflowExecutionAndTemplate(namespace string, uid string) (workflow *WorkflowExecution, err error) {
-	sb := sb.Select(getWorkflowExecutionColumns("we", "")...).
+	sb := sb.Select(getWorkflowExecutionColumns("we")...).
 		Columns(getWorkflowTemplateColumns("wt", "workflow_template")...).
 		Columns(`wtv.manifest "workflow_template.manifest"`, `wtv.version "workflow_template.version"`).
 		From("workflow_executions we").

pkg/workspace_template.go

@@ -401,7 +401,7 @@ func createStatefulSetManifest(spec *WorkspaceSpec, config map[string]string, se
 		env.PrependEnvVarToContainer(container, "ONEPANEL_DOMAIN", config["ONEPANEL_DOMAIN"])
 		env.PrependEnvVarToContainer(container, "ONEPANEL_PROVIDER", config["ONEPANEL_PROVIDER"])
 		env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_NAMESPACE", "{{workflow.namespace}}")
-		env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_UID", "{{workflow.parameters.sys-name}}")
+		env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_UID", "{{workflow.parameters.sys-uid}}")
 
 		for _, service := range services {
 			envName := fmt.Sprintf("ONEPANEL_SERVICES_%v_API_URL", strings.ToUpper(service.Name))

server/auth/auth.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"github.com/onepanelio/core/api"
 	"github.com/onepanelio/core/pkg/util"
-	"github.com/onepanelio/core/pkg/util/tokens"
 	v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"net/http"
 	"strings"
@@ -41,6 +40,9 @@ func getBearerToken(ctx context.Context) (*string, bool) {
 			return nil, false
 		}
 		t = strings.ReplaceAll(t, prefix, "")
+		if t == "null" {
+			return nil, false
+		}
 		return &t, true
 	}
 
@@ -69,15 +71,11 @@ func getClient(ctx context.Context, kubeConfig *v1.Config, db *v1.DB, sysConfig
 		return nil, status.Error(codes.Unauthenticated, `Missing or invalid "authorization" header.`)
 	}
 
-	tokenContent, err := tokens.ParseJWTToken(*bearerToken, sysConfig.HMACKey())
-	if err != nil {
-		return nil, err
+	if sysConfig["token"] != *bearerToken {
+		sysConfig["token"] = *bearerToken
 	}
 
-	sysConfig["jwtToken"] = *bearerToken
-	sysConfig["jwtUsername"] = tokenContent.Username
-
-	kubeConfig.BearerToken = tokenContent.Token
+	kubeConfig.BearerToken = *bearerToken
 
 	client, err := v1.NewClient(kubeConfig, db, sysConfig)
 	if err != nil {
@@ -167,7 +165,7 @@ func UnaryInterceptor(kubeConfig *v1.Config, db *v1.DB, sysConfig v1.SystemConfi
 				return resp, errors.New("LogInRequest does not have correct request type")
 			}
 
-			defaultClient, err := v1.GetDefaultClient()
+			defaultClient, err := v1.GetDefaultClientWithDB(db)
 			if err != nil {
 				return nil, err
 			}
@@ -182,17 +180,7 @@ func UnaryInterceptor(kubeConfig *v1.Config, db *v1.DB, sysConfig v1.SystemConfi
 				return nil, err
 			}
 
-			hmac := sysConfig.HMACKey()
-			if len(hmac) == 0 {
-				return nil, errors.New("HMAC key not found in secrets - this value is required")
-			}
-
-			jwtToken, err := tokens.CreateJWTToken(tokenRequest.Username, rawToken, hmac)
-			if err != nil {
-				return nil, err
-			}
-
-			md.Set("onepanel-auth-token", jwtToken)
+			md.Set("onepanel-auth-token", rawToken)
 
 			ctx, err = getClient(ctx, kubeConfig, db, sysConfig)
 			if err != nil {

server/auth_server.go

@@ -67,9 +67,8 @@ func (a *AuthServer) IsValidToken(ctx context.Context, req *api.IsValidTokenRequ
 		return
 	}
 	res = &api.IsValidTokenResponse{
-		Domain:   config["ONEPANEL_DOMAIN"],
-		JwtToken: config["jwtToken"],
-		Username: config["jwtUsername"],
+		Domain: config["ONEPANEL_DOMAIN"],
+		Token:  config["token"],
 	}
 
 	return res, nil

server/label_server.go

@@ -7,21 +7,22 @@ import (
 	"github.com/onepanelio/core/server/auth"
 )
 
-func resourceIdentifierToArgoResource(identifier string) string {
+func getGroupAndResourceByIdentifier(identifier string) (group, resource string) {
+	group = "argoproj.io"
 	switch identifier {
 	case v1.TypeWorkflowTemplate:
-		return "workflowtemplates"
+		return group, "workflowtemplates"
 	case v1.TypeWorkflowTemplateVersion:
-		return "workflowtemplates"
+		return group, "workflowtemplates"
 	case v1.TypeWorkflowExecution:
-		return "workflows"
+		return group, "workflows"
 	case v1.TypeCronWorkflow:
-		return "cronworkflows"
+		return group, "cronworkflows"
 	case v1.TypeWorkspace:
-		return "statefulset"
+		return "onepanel.io", "workspaces"
 	}
 
-	return ""
+	return "", ""
 }
 
 func mapLabelsToKeyValue(labels []*v1.Label) []*api.KeyValue {
@@ -54,10 +55,10 @@ func NewLabelServer() *LabelServer {
 }
 
 func (s *LabelServer) GetLabels(ctx context.Context, req *api.GetLabelsRequest) (*api.GetLabelsResponse, error) {
-	argoResource := resourceIdentifierToArgoResource(req.Resource)
+	group, resource := getGroupAndResourceByIdentifier(req.Resource)
 
 	client := getClient(ctx)
-	allowed, err := auth.IsAuthorized(client, req.Namespace, "get", "argoproj.io", argoResource, "")
+	allowed, err := auth.IsAuthorized(client, req.Namespace, "get", group, resource, "")
 	if err != nil || !allowed {
 		return nil, err
 	}
@@ -73,10 +74,10 @@ func (s *LabelServer) GetLabels(ctx context.Context, req *api.GetLabelsRequest)
 }
 
 func (s *LabelServer) AddLabels(ctx context.Context, req *api.AddLabelsRequest) (*api.GetLabelsResponse, error) {
-	argoResource := resourceIdentifierToArgoResource(req.Resource)
+	group, resource := getGroupAndResourceByIdentifier(req.Resource)
 
 	client := getClient(ctx)
-	allowed, err := auth.IsAuthorized(client, req.Namespace, "create", "argoproj.io", argoResource, "")
+	allowed, err := auth.IsAuthorized(client, req.Namespace, "create", group, resource, "")
 	if err != nil || !allowed {
 		return nil, err
 	}
@@ -97,10 +98,10 @@ func (s *LabelServer) AddLabels(ctx context.Context, req *api.AddLabelsRequest)
 }
 
 func (s *LabelServer) ReplaceLabels(ctx context.Context, req *api.ReplaceLabelsRequest) (*api.GetLabelsResponse, error) {
-	argoResource := resourceIdentifierToArgoResource(req.Resource)
+	group, resource := getGroupAndResourceByIdentifier(req.Resource)
 
 	client := getClient(ctx)
-	allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "argoproj.io", argoResource, "")
+	allowed, err := auth.IsAuthorized(client, req.Namespace, "update", group, resource, "")
 	if err != nil || !allowed {
 		return nil, err
 	}
@@ -121,11 +122,11 @@ func (s *LabelServer) ReplaceLabels(ctx context.Context, req *api.ReplaceLabelsR
 }
 
 func (s *LabelServer) DeleteLabel(ctx context.Context, req *api.DeleteLabelRequest) (*api.GetLabelsResponse, error) {
-	argoResource := resourceIdentifierToArgoResource(req.Resource)
+	group, resource := getGroupAndResourceByIdentifier(req.Resource)
 
 	client := getClient(ctx)
 	// update verb here since we are not deleting the resource, but labels
-	allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "argoproj.io", argoResource, "")
+	allowed, err := auth.IsAuthorized(client, req.Namespace, "update", group, resource, "")
 	if err != nil || !allowed {
 		return nil, err
 	}