mirror of
https://github.com/onepanelio/onepanel.git
synced 2025-09-27 01:56:03 +08:00
feat: updated go migration data to have metadata to make it easier to get all of the information from one file
This commit is contained in:
Andrey Melnikov
@@ -3,7 +3,9 @@ package migration
|
||||
import (
|
||||
"fmt"
|
||||
v1 "github.com/onepanelio/core/pkg"
|
||||
"github.com/onepanelio/core/pkg/util/data"
|
||||
uid2 "github.com/onepanelio/core/pkg/util/uid"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// createWorkspaceTemplate will create the workspace template given by {{templateName}} with the contents
|
||||
@@ -21,7 +23,13 @@ func createWorkspaceTemplate(filename, templateName, description string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
newManifest, err := readDataFile(filename)
|
||||
filename = filepath.Join("db", "yaml", filename)
|
||||
manifestFile, err := data.ManifestFileFromFile(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newManifest, err := manifestFile.SpecString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -97,12 +105,19 @@ func updateWorkspaceTemplateManifest(filename, templateName string) error {
|
||||
}
|
||||
defer client.DB.Close()
|
||||
|
||||
filename = filepath.Join("db", "yaml", filename)
|
||||
|
||||
namespaces, err := client.ListOnepanelEnabledNamespaces()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newManifest, err := readDataFile(filename)
|
||||
manifest, err := data.ManifestFileFromFile(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newManifest, err := manifest.SpecString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -145,7 +160,14 @@ func createWorkflowTemplate(filename, templateName string, labels map[string]str
|
||||
return err
|
||||
}
|
||||
|
||||
manifest, err := readDataFile(filename)
|
||||
filename = filepath.Join("db", "yaml", filename)
|
||||
|
||||
manifestFile, err := data.ManifestFileFromFile(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifest, err := manifestFile.SpecString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -190,7 +212,14 @@ func updateWorkflowTemplateManifest(filename, templateName string, labels map[st
|
||||
return err
|
||||
}
|
||||
|
||||
newManifest, err := readDataFile(filename)
|
||||
filename = filepath.Join("db", "yaml", filename)
|
||||
|
||||
manifestFile, err := data.ManifestFileFromFile(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newManifest, err := manifestFile.SpecString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,4 +1,15 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/nni-hyperparameter-tuning/mnist/
|
||||
metadata:
|
||||
name: "Hyperparameter Tuning Example"
|
||||
kind: Workflow
|
||||
version: 20201225172926
|
||||
action: create
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/nni-hyperparameter-tuning/mnist/"
|
||||
deprecated: true
|
||||
labels:
|
||||
framework: tensorflow
|
||||
tuner: TPE
|
||||
"created-by": system
|
||||
spec:
|
||||
entrypoint: main
|
||||
arguments:
|
||||
parameters:
|
||||
@@ -46,7 +57,7 @@ arguments:
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.nodepool
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
required: true
|
||||
|
||||
volumeClaimTemplates:
|
||||
|
||||
@@ -1,4 +1,15 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/nni-hyperparameter-tuning/mnist/
|
||||
metadata:
|
||||
name: "Hyperparameter Tuning Example"
|
||||
kind: Workflow
|
||||
version: 20210118175809
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/nni-hyperparameter-tuning/mnist/"
|
||||
deprecated: true
|
||||
labels:
|
||||
framework: tensorflow
|
||||
tuner: TPE
|
||||
"created-by": system
|
||||
spec:
|
||||
# Workflow Template example for hyperparameter tuning
|
||||
# Documentation: https://docs.onepanel.ai/docs/reference/workflows/hyperparameter-tuning
|
||||
#
|
||||
@@ -65,7 +76,7 @@ arguments:
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.nodepool
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
required: true
|
||||
|
||||
templates:
|
||||
@@ -130,7 +141,7 @@ templates:
|
||||
- name: hyperparamtuning-output
|
||||
mountPath: /mnt/output
|
||||
nodeSelector:
|
||||
{{.NodePoolLabel}}: '{{workflow.parameters.sys-node-pool}}'
|
||||
"{{.NodePoolLabel}}": '{{workflow.parameters.sys-node-pool}}'
|
||||
sidecars:
|
||||
- name: nni-web-ui
|
||||
image: onepanel/nni-web-ui:0.17.0
|
||||
|
||||
197
db/yaml/workflows/maskrcnn-training/20200812104328.yaml
Normal file
197
db/yaml/workflows/maskrcnn-training/20200812104328.yaml
Normal file
@@ -0,0 +1,197 @@
|
||||
metadata:
|
||||
name: "MaskRCNN Training"
|
||||
kind: Workflow
|
||||
version: 20200812104328
|
||||
action: create
|
||||
labels:
|
||||
"used-by": "cvat"
|
||||
"created-by": "system"
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: source
|
||||
value: https://github.com/onepanelio/Mask_RCNN.git
|
||||
displayName: Model source code
|
||||
type: hidden
|
||||
visibility: private
|
||||
|
||||
- name: sys-annotation-path
|
||||
value: annotation-dump/sample_dataset
|
||||
hint: Path to annotated data in default object storage (i.e S3). In CVAT, this parameter will be pre-populated.
|
||||
displayName: Dataset path
|
||||
visibility: private
|
||||
|
||||
- name: sys-output-path
|
||||
value: workflow-data/output/sample_output
|
||||
hint: Path to store output artifacts in default object storage (i.e s3). In CVAT, this parameter will be pre-populated.
|
||||
displayName: Workflow output path
|
||||
visibility: private
|
||||
|
||||
- name: sys-finetune-checkpoint
|
||||
value: ''
|
||||
hint: Select the last fine-tune checkpoint for this model. It may take up to 5 minutes for a recent checkpoint show here. Leave empty if this is the first time you're training this model.
|
||||
displayName: Checkpoint path
|
||||
visibility: public
|
||||
|
||||
- name: sys-num-classes
|
||||
displayName: Number of classes
|
||||
hint: Number of classes (i.e in CVAT taks) + 1 for background
|
||||
value: '81'
|
||||
visibility: private
|
||||
|
||||
- name: extras
|
||||
displayName: Hyperparameters
|
||||
visibility: public
|
||||
type: textarea.textarea
|
||||
value: |-
|
||||
stage-1-epochs=1 # Epochs for network heads
|
||||
stage-2-epochs=2 # Epochs for finetune layers
|
||||
stage-3-epochs=3 # Epochs for all layers
|
||||
hint: "Please refer to our <a href='https://docs.onepanel.ai/docs/getting-started/use-cases/computervision/annotation/cvat/cvat_annotation_model#arguments-optional' target='_blank'>documentation</a> for more information on parameters."
|
||||
|
||||
- name: dump-format
|
||||
type: select.select
|
||||
value: cvat_coco
|
||||
displayName: CVAT dump format
|
||||
visibility: public
|
||||
options:
|
||||
- name: 'MS COCO'
|
||||
value: 'cvat_coco'
|
||||
- name: 'TF Detection API'
|
||||
value: 'cvat_tfrecord'
|
||||
|
||||
- name: tf-image
|
||||
visibility: public
|
||||
value: tensorflow/tensorflow:1.13.1-py3
|
||||
type: select.select
|
||||
displayName: Select TensorFlow image
|
||||
hint: Select the GPU image if you are running on a GPU node pool
|
||||
options:
|
||||
- name: 'TensorFlow 1.13.1 CPU Image'
|
||||
value: 'tensorflow/tensorflow:1.13.1-py3'
|
||||
- name: 'TensorFlow 1.13.1 GPU Image'
|
||||
value: 'tensorflow/tensorflow:1.13.1-gpu-py3'
|
||||
|
||||
- displayName: Node pool
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.select
|
||||
visibility: public
|
||||
name: sys-node-pool
|
||||
value: Standard_D4s_v3
|
||||
required: true
|
||||
options:
|
||||
- name: 'CPU: 2, RAM: 8GB'
|
||||
value: Standard_D2s_v3
|
||||
- name: 'CPU: 4, RAM: 16GB'
|
||||
value: Standard_D4s_v3
|
||||
- name: 'GPU: 1xK80, CPU: 6, RAM: 56GB'
|
||||
value: Standard_NC6
|
||||
|
||||
entrypoint: main
|
||||
templates:
|
||||
- dag:
|
||||
tasks:
|
||||
- name: train-model
|
||||
template: tensorflow
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
# - arguments:
|
||||
# artifacts:
|
||||
# - from: '{{tasks.train-model.outputs.artifacts.sys-metrics}}'
|
||||
# name: metrics
|
||||
# parameters:
|
||||
# - name: status
|
||||
# value: '{{tasks.train-model.status}}'
|
||||
# dependencies:
|
||||
# - train-model
|
||||
# name: notify-in-slack
|
||||
# template: slack-notify-success
|
||||
name: main
|
||||
- container:
|
||||
args:
|
||||
- |
|
||||
apt-get update \
|
||||
&& apt-get install -y git wget libglib2.0-0 libsm6 libxext6 libxrender-dev \
|
||||
&& pip install -r requirements.txt \
|
||||
&& pip install boto3 pyyaml google-cloud-storage \
|
||||
&& git clone https://github.com/waleedka/coco \
|
||||
&& cd coco/PythonAPI \
|
||||
&& python setup.py build_ext install \
|
||||
&& rm -rf build \
|
||||
&& cd ../../ \
|
||||
&& wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 \
|
||||
&& python setup.py install && ls \
|
||||
&& python samples/coco/cvat.py train --dataset=/mnt/data/datasets \
|
||||
--model=workflow_maskrcnn \
|
||||
--extras="{{workflow.parameters.extras}}" \
|
||||
--ref_model_path="{{workflow.parameters.sys-finetune-checkpoint}}" \
|
||||
--num_classes="{{workflow.parameters.sys-num-classes}}" \
|
||||
&& cd /mnt/src/ \
|
||||
&& python prepare_dataset.py /mnt/data/datasets/annotations/instances_default.json
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
image: '{{workflow.parameters.tf-image}}'
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/data
|
||||
name: data
|
||||
- mountPath: /mnt/output
|
||||
name: output
|
||||
workingDir: /mnt/src
|
||||
nodeSelector:
|
||||
beta.kubernetes.io/instance-type: '{{workflow.parameters.sys-node-pool}}'
|
||||
inputs:
|
||||
artifacts:
|
||||
- name: data
|
||||
path: /mnt/data/datasets/
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.sys-annotation-path}}'
|
||||
- git:
|
||||
repo: '{{workflow.parameters.source}}'
|
||||
revision: "no-boto"
|
||||
name: src
|
||||
path: /mnt/src
|
||||
name: tensorflow
|
||||
outputs:
|
||||
artifacts:
|
||||
- name: model
|
||||
optional: true
|
||||
path: /mnt/output
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.sys-output-path}}'
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#- container:
|
||||
# args:
|
||||
# - SLACK_USERNAME=Onepanel SLACK_TITLE="{{workflow.name}} {{inputs.parameters.status}}"
|
||||
# SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd
|
||||
# SLACK_MESSAGE=$(cat /tmp/metrics.json)} ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
# image: technosophos/slack-notify
|
||||
# inputs:
|
||||
# artifacts:
|
||||
# - name: metrics
|
||||
# optional: true
|
||||
# path: /tmp/metrics.json
|
||||
# parameters:
|
||||
# - name: status
|
||||
# name: slack-notify-success
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
creationTimestamp: null
|
||||
name: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
- metadata:
|
||||
creationTimestamp: null
|
||||
name: output
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
191
db/yaml/workflows/maskrcnn-training/20200824095513.yaml
Normal file
191
db/yaml/workflows/maskrcnn-training/20200824095513.yaml
Normal file
@@ -0,0 +1,191 @@
|
||||
metadata:
|
||||
name: "MaskRCNN Training"
|
||||
kind: Workflow
|
||||
version: 20200824095513
|
||||
action: update
|
||||
labels:
|
||||
"used-by": "cvat"
|
||||
"created-by": "system"
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: source
|
||||
value: https://github.com/onepanelio/Mask_RCNN.git
|
||||
displayName: Model source code
|
||||
type: hidden
|
||||
visibility: private
|
||||
|
||||
- name: cvat-annotation-path
|
||||
value: annotation-dump/sample_dataset
|
||||
hint: Path to annotated data in default object storage (i.e S3). In CVAT, this parameter will be pre-populated.
|
||||
displayName: Dataset path
|
||||
visibility: private
|
||||
|
||||
- name: cvat-output-path
|
||||
value: workflow-data/output/sample_output
|
||||
hint: Path to store output artifacts in default object storage (i.e s3). In CVAT, this parameter will be pre-populated.
|
||||
displayName: Workflow output path
|
||||
visibility: private
|
||||
|
||||
- name: cvat-finetune-checkpoint
|
||||
value: ''
|
||||
hint: Select the last fine-tune checkpoint for this model. It may take up to 5 minutes for a recent checkpoint show here. Leave empty if this is the first time you're training this model.
|
||||
displayName: Checkpoint path
|
||||
visibility: public
|
||||
|
||||
- name: cvat-num-classes
|
||||
displayName: Number of classes
|
||||
hint: Number of classes (i.e in CVAT taks) + 1 for background
|
||||
value: '81'
|
||||
visibility: private
|
||||
|
||||
- name: hyperparameters
|
||||
displayName: Hyperparameters
|
||||
visibility: public
|
||||
type: textarea.textarea
|
||||
value: |-
|
||||
stage-1-epochs=1 # Epochs for network heads
|
||||
stage-2-epochs=2 # Epochs for finetune layers
|
||||
stage-3-epochs=3 # Epochs for all layers
|
||||
hint: "Please refer to our <a href='https://docs.onepanel.ai/docs/getting-started/use-cases/computervision/annotation/cvat/cvat_annotation_model#arguments-optional' target='_blank'>documentation</a> for more information on parameters. Number of classes will be automatically populated if you had 'sys-num-classes' parameter in a workflow."
|
||||
|
||||
- name: dump-format
|
||||
value: cvat_coco
|
||||
displayName: CVAT dump format
|
||||
visibility: public
|
||||
|
||||
- name: tf-image
|
||||
visibility: public
|
||||
value: tensorflow/tensorflow:1.13.1-py3
|
||||
type: select.select
|
||||
displayName: Select TensorFlow image
|
||||
hint: Select the GPU image if you are running on a GPU node pool
|
||||
options:
|
||||
- name: 'TensorFlow 1.13.1 CPU Image'
|
||||
value: 'tensorflow/tensorflow:1.13.1-py3'
|
||||
- name: 'TensorFlow 1.13.1 GPU Image'
|
||||
value: 'tensorflow/tensorflow:1.13.1-gpu-py3'
|
||||
|
||||
- displayName: Node pool
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.select
|
||||
visibility: public
|
||||
name: sys-node-pool
|
||||
value: Standard_D4s_v3
|
||||
required: true
|
||||
options:
|
||||
- name: 'CPU: 2, RAM: 8GB'
|
||||
value: Standard_D2s_v3
|
||||
- name: 'CPU: 4, RAM: 16GB'
|
||||
value: Standard_D4s_v3
|
||||
- name: 'GPU: 1xK80, CPU: 6, RAM: 56GB'
|
||||
value: Standard_NC6
|
||||
|
||||
entrypoint: main
|
||||
templates:
|
||||
- dag:
|
||||
tasks:
|
||||
- name: train-model
|
||||
template: tensorflow
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
# - arguments:
|
||||
# artifacts:
|
||||
# - from: '{{tasks.train-model.outputs.artifacts.sys-metrics}}'
|
||||
# name: metrics
|
||||
# parameters:
|
||||
# - name: status
|
||||
# value: '{{tasks.train-model.status}}'
|
||||
# dependencies:
|
||||
# - train-model
|
||||
# name: notify-in-slack
|
||||
# template: slack-notify-success
|
||||
name: main
|
||||
- container:
|
||||
args:
|
||||
- |
|
||||
apt-get update \
|
||||
&& apt-get install -y git wget libglib2.0-0 libsm6 libxext6 libxrender-dev \
|
||||
&& pip install -r requirements.txt \
|
||||
&& pip install boto3 pyyaml google-cloud-storage \
|
||||
&& git clone https://github.com/waleedka/coco \
|
||||
&& cd coco/PythonAPI \
|
||||
&& python setup.py build_ext install \
|
||||
&& rm -rf build \
|
||||
&& cd ../../ \
|
||||
&& wget https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5 \
|
||||
&& python setup.py install && ls \
|
||||
&& python samples/coco/cvat.py train --dataset=/mnt/data/datasets \
|
||||
--model=workflow_maskrcnn \
|
||||
--extras="{{workflow.parameters.hyperparameters}}" \
|
||||
--ref_model_path="{{workflow.parameters.cvat-finetune-checkpoint}}" \
|
||||
--num_classes="{{workflow.parameters.cvat-num-classes}}" \
|
||||
&& cd /mnt/src/ \
|
||||
&& python prepare_dataset.py /mnt/data/datasets/annotations/instances_default.json
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
image: '{{workflow.parameters.tf-image}}'
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/data
|
||||
name: data
|
||||
- mountPath: /mnt/output
|
||||
name: output
|
||||
workingDir: /mnt/src
|
||||
nodeSelector:
|
||||
beta.kubernetes.io/instance-type: '{{workflow.parameters.sys-node-pool}}'
|
||||
inputs:
|
||||
artifacts:
|
||||
- name: data
|
||||
path: /mnt/data/datasets/
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-annotation-path}}'
|
||||
- git:
|
||||
repo: '{{workflow.parameters.source}}'
|
||||
revision: "no-boto"
|
||||
name: src
|
||||
path: /mnt/src
|
||||
name: tensorflow
|
||||
outputs:
|
||||
artifacts:
|
||||
- name: model
|
||||
optional: true
|
||||
path: /mnt/output
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-output-path}}/{{workflow.name}}'
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#- container:
|
||||
# args:
|
||||
# - SLACK_USERNAME=Onepanel SLACK_TITLE="{{workflow.name}} {{inputs.parameters.status}}"
|
||||
# SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd
|
||||
# SLACK_MESSAGE=$(cat /tmp/metrics.json)} ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
# image: technosophos/slack-notify
|
||||
# inputs:
|
||||
# artifacts:
|
||||
# - name: metrics
|
||||
# optional: true
|
||||
# path: /tmp/metrics.json
|
||||
# parameters:
|
||||
# - name: status
|
||||
# name: slack-notify-success
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
creationTimestamp: null
|
||||
name: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
- metadata:
|
||||
creationTimestamp: null
|
||||
name: output
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
@@ -1,3 +1,12 @@
|
||||
metadata:
|
||||
name: "MaskRCNN Training"
|
||||
kind: Workflow
|
||||
version: 20201115145814
|
||||
action: update
|
||||
labels:
|
||||
"used-by": "cvat"
|
||||
"created-by": "system"
|
||||
spec:
|
||||
entrypoint: main
|
||||
arguments:
|
||||
parameters:
|
||||
@@ -137,7 +146,7 @@ templates:
|
||||
artifacts:
|
||||
- name: data
|
||||
path: /mnt/data/datasets/
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-annotation-path}}'
|
||||
- git:
|
||||
repo: '{{workflow.parameters.source}}'
|
||||
@@ -149,7 +158,7 @@ templates:
|
||||
- name: model
|
||||
optional: true
|
||||
path: /mnt/output
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-output-path}}/{{workflow.name}}'
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#- container:
|
||||
|
||||
@@ -1,3 +1,12 @@
|
||||
metadata:
|
||||
name: "MaskRCNN Training"
|
||||
kind: Workflow
|
||||
version: 20201208155115
|
||||
action: update
|
||||
labels:
|
||||
"used-by": "cvat"
|
||||
"created-by": "system"
|
||||
spec:
|
||||
entrypoint: main
|
||||
arguments:
|
||||
parameters:
|
||||
@@ -139,7 +148,7 @@ templates:
|
||||
artifacts:
|
||||
- name: data
|
||||
path: /mnt/data/datasets/
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-annotation-path}}'
|
||||
- git:
|
||||
repo: '{{workflow.parameters.source}}'
|
||||
@@ -151,7 +160,7 @@ templates:
|
||||
- name: model
|
||||
optional: true
|
||||
path: /mnt/output
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-output-path}}/{{workflow.name}}'
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#- container:
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/maskrcnn-training/
|
||||
metadata:
|
||||
name: "MaskRCNN Training"
|
||||
kind: Workflow
|
||||
version: 20201221195937
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/maskrcnn-training/"
|
||||
labels:
|
||||
"used-by": "cvat"
|
||||
"created-by": "system"
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: cvat-annotation-path
|
||||
@@ -57,7 +66,7 @@ arguments:
|
||||
type: select.nodepool
|
||||
visibility: public
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
required: true
|
||||
|
||||
entrypoint: main
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/maskrcnn-training/
|
||||
metadata:
|
||||
name: "MaskRCNN Training"
|
||||
kind: Workflow
|
||||
version: 20210118175809
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/maskrcnn-training/"
|
||||
labels:
|
||||
"used-by": "cvat"
|
||||
"created-by": "system"
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: cvat-annotation-path
|
||||
@@ -71,7 +80,7 @@ arguments:
|
||||
type: select.nodepool
|
||||
visibility: public
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
required: true
|
||||
|
||||
entrypoint: main
|
||||
@@ -122,7 +131,7 @@ templates:
|
||||
- containerPort: 6006
|
||||
name: tensorboard
|
||||
nodeSelector:
|
||||
{{.NodePoolLabel}}: '{{workflow.parameters.sys-node-pool}}'
|
||||
"{{.NodePoolLabel}}": '{{workflow.parameters.sys-node-pool}}'
|
||||
inputs:
|
||||
artifacts:
|
||||
- name: data
|
||||
@@ -163,7 +172,7 @@ templates:
|
||||
name: processed-data
|
||||
workingDir: /mnt/src
|
||||
nodeSelector:
|
||||
{{.NodePoolLabel}}: '{{workflow.parameters.sys-node-pool}}'
|
||||
"{{.NodePoolLabel}}": '{{workflow.parameters.sys-node-pool}}'
|
||||
inputs:
|
||||
artifacts:
|
||||
- name: data
|
||||
|
||||
@@ -1,3 +1,12 @@
|
||||
metadata:
|
||||
name: "PyTorch Training"
|
||||
kind: Workflow
|
||||
version: 20200605090509
|
||||
action: create
|
||||
labels:
|
||||
"created-by": "system"
|
||||
framework: pytorch
|
||||
spec:
|
||||
entrypoint: main
|
||||
arguments:
|
||||
parameters:
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/pytorch-mnist-training/
|
||||
metadata:
|
||||
name: "PyTorch Training"
|
||||
kind: Workflow
|
||||
version: 20201221194344
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/pytorch-mnist-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
framework: pytorch
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: epochs
|
||||
@@ -7,7 +16,7 @@ arguments:
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.nodepool
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
visibility: public
|
||||
required: true
|
||||
entrypoint: main
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/pytorch-mnist-training/
|
||||
metadata:
|
||||
name: "PyTorch Training"
|
||||
kind: Workflow
|
||||
version: 20210118175809
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/pytorch-mnist-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
framework: pytorch
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: epochs
|
||||
@@ -7,7 +16,7 @@ arguments:
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.nodepool
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
visibility: public
|
||||
required: true
|
||||
entrypoint: main
|
||||
@@ -173,7 +182,7 @@ templates:
|
||||
- name: output
|
||||
mountPath: /mnt/output
|
||||
nodeSelector:
|
||||
{{.NodePoolLabel}}: '{{workflow.parameters.sys-node-pool}}'
|
||||
"{{.NodePoolLabel}}": '{{workflow.parameters.sys-node-pool}}'
|
||||
sidecars:
|
||||
- name: tensorboard
|
||||
image: onepanel/dl:0.17.0
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/pytorch-mnist-training/
|
||||
metadata:
|
||||
name: "PyTorch Training"
|
||||
kind: Workflow
|
||||
version: 20210323175655
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/pytorch-mnist-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
framework: pytorch
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: epochs
|
||||
@@ -7,7 +16,7 @@ arguments:
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.nodepool
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
visibility: public
|
||||
required: true
|
||||
entrypoint: main
|
||||
@@ -173,7 +182,7 @@ templates:
|
||||
- name: output
|
||||
mountPath: /mnt/output
|
||||
nodeSelector:
|
||||
{{.NodePoolLabel}}: '{{workflow.parameters.sys-node-pool}}'
|
||||
"{{.NodePoolLabel}}": '{{workflow.parameters.sys-node-pool}}'
|
||||
sidecars:
|
||||
- name: tensorboard
|
||||
image: onepanel/dl:v0.20.0
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/template.yaml
|
||||
metadata:
|
||||
name: "TensorFlow Training"
|
||||
kind: Workflow
|
||||
version: 20200605090535
|
||||
action: create
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/template.yaml"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
framework: tensorflow
|
||||
spec:
|
||||
entrypoint: main
|
||||
arguments:
|
||||
parameters:
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/template.yaml
|
||||
metadata:
|
||||
name: "TensorFlow Training"
|
||||
kind: Workflow
|
||||
version: 20201209124226
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/template.yaml"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
framework: tensorflow
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: epochs
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/
|
||||
metadata:
|
||||
name: "TensorFlow Training"
|
||||
kind: Workflow
|
||||
version: 20201223062947
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
framework: tensorflow
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: epochs
|
||||
@@ -7,7 +16,7 @@ arguments:
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.nodepool
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
visibility: public
|
||||
required: true
|
||||
entrypoint: main
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/
|
||||
metadata:
|
||||
name: "TensorFlow Training"
|
||||
kind: Workflow
|
||||
version: 20210118175809
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
framework: tensorflow
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: epochs
|
||||
@@ -7,7 +16,7 @@ arguments:
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.nodepool
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
visibility: public
|
||||
required: true
|
||||
entrypoint: main
|
||||
@@ -84,7 +93,7 @@ templates:
|
||||
- name: output
|
||||
mountPath: /mnt/output
|
||||
nodeSelector:
|
||||
{{.NodePoolLabel}}: '{{workflow.parameters.sys-node-pool}}'
|
||||
"{{.NodePoolLabel}}": '{{workflow.parameters.sys-node-pool}}'
|
||||
sidecars:
|
||||
- name: tensorboard
|
||||
image: onepanel/dl:0.17.0
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/
|
||||
metadata:
|
||||
name: "TensorFlow Training"
|
||||
kind: Workflow
|
||||
version: 20210323175655
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
framework: tensorflow
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: epochs
|
||||
@@ -7,7 +16,7 @@ arguments:
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.nodepool
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
visibility: public
|
||||
required: true
|
||||
entrypoint: main
|
||||
@@ -84,7 +93,7 @@ templates:
|
||||
- name: output
|
||||
mountPath: /mnt/output
|
||||
nodeSelector:
|
||||
{{.NodePoolLabel}}: '{{workflow.parameters.sys-node-pool}}'
|
||||
"{{.NodePoolLabel}}": '{{workflow.parameters.sys-node-pool}}'
|
||||
sidecars:
|
||||
- name: tensorboard
|
||||
image: onepanel/dl:v0.20.0
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
metadata:
|
||||
name: "TF Object Detection Training"
|
||||
kind: Workflow
|
||||
version: 20200812104328
|
||||
action: create
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tf-object-detection-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
"used-by": "cvat"
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: source
|
||||
value: https://github.com/tensorflow/models.git
|
||||
displayName: Model source code
|
||||
type: hidden
|
||||
visibility: private
|
||||
|
||||
- name: trainingsource
|
||||
value: https://github.com/onepanelio/cvat-training.git
|
||||
type: hidden
|
||||
visibility: private
|
||||
|
||||
- name: revision
|
||||
value: v1.13.0
|
||||
type: hidden
|
||||
visibility: private
|
||||
|
||||
- name: sys-annotation-path
|
||||
value: annotation-dump/sample_dataset
|
||||
displayName: Dataset path
|
||||
hint: Path to annotated data in default object storage (i.e S3). In CVAT, this parameter will be pre-populated.
|
||||
|
||||
- name: sys-output-path
|
||||
value: workflow-data/output/sample_output
|
||||
hint: Path to store output artifacts in default object storage (i.e s3). In CVAT, this parameter will be pre-populated.
|
||||
displayName: Workflow output path
|
||||
visibility: private
|
||||
|
||||
- name: ref-model
|
||||
value: frcnn-res50-coco
|
||||
displayName: Model
|
||||
hint: TF Detection API's model to use for training.
|
||||
type: select.select
|
||||
visibility: public
|
||||
options:
|
||||
- name: 'Faster RCNN-ResNet 101-COCO'
|
||||
value: frcnn-res101-coco
|
||||
- name: 'Faster RCNN-ResNet 101-Low Proposal-COCO'
|
||||
value: frcnn-res101-low
|
||||
- name: 'Faster RCNN-ResNet 50-COCO'
|
||||
value: frcnn-res50-coco
|
||||
- name: 'Faster RCNN-NAS-COCO'
|
||||
value: frcnn-nas-coco
|
||||
- name: 'SSD MobileNet V1-COCO'
|
||||
value: ssd-mobilenet-v1-coco2
|
||||
- name: 'SSD MobileNet V2-COCO'
|
||||
value: ssd-mobilenet-v2-coco
|
||||
- name: 'SSDLite MobileNet-COCO'
|
||||
value: ssdlite-mobilenet-coco
|
||||
|
||||
- name: extras
|
||||
value: |-
|
||||
epochs=1000
|
||||
displayName: Hyperparameters
|
||||
visibility: public
|
||||
type: textarea.textarea
|
||||
hint: "Please refer to our <a href='https://docs.onepanel.ai/docs/getting-started/use-cases/computervision/annotation/cvat/cvat_annotation_model#arguments-optional' target='_blank'>documentation</a> for more information on parameters. Number of classes will be automatically populated if you had 'sys-num-classes' parameter in a workflow."
|
||||
|
||||
- name: sys-finetune-checkpoint
|
||||
value: ''
|
||||
hint: Select the last fine-tune checkpoint for this model. It may take up to 5 minutes for a recent checkpoint show here. Leave empty if this is the first time you're training this model.
|
||||
displayName: Checkpoint path
|
||||
visibility: public
|
||||
|
||||
- name: sys-num-classes
|
||||
value: '81'
|
||||
hint: Number of classes
|
||||
displayName: Number of classes
|
||||
visibility: private
|
||||
|
||||
- name: tf-image
|
||||
value: tensorflow/tensorflow:1.13.1-py3
|
||||
type: select.select
|
||||
displayName: Select TensorFlow image
|
||||
visibility: public
|
||||
hint: Select the GPU image if you are running on a GPU node pool
|
||||
options:
|
||||
- name: 'TensorFlow 1.13.1 CPU Image'
|
||||
value: 'tensorflow/tensorflow:1.13.1-py3'
|
||||
- name: 'TensorFlow 1.13.1 GPU Image'
|
||||
value: 'tensorflow/tensorflow:1.13.1-gpu-py3'
|
||||
|
||||
- displayName: Node pool
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.select
|
||||
name: sys-node-pool
|
||||
value: Standard_D4s_v3
|
||||
visibility: public
|
||||
required: true
|
||||
options:
|
||||
- name: 'CPU: 2, RAM: 8GB'
|
||||
value: Standard_D2s_v3
|
||||
- name: 'CPU: 4, RAM: 16GB'
|
||||
value: Standard_D4s_v3
|
||||
- name: 'GPU: 1xK80, CPU: 6, RAM: 56GB'
|
||||
value: Standard_NC6
|
||||
- name: dump-format
|
||||
value: cvat_tfrecord
|
||||
visibility: public
|
||||
entrypoint: main
|
||||
templates:
|
||||
- dag:
|
||||
tasks:
|
||||
- name: train-model
|
||||
template: tensorflow
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
# - arguments:
|
||||
# artifacts:
|
||||
# - from: '{{tasks.train-model.outputs.artifacts.sys-metrics}}'
|
||||
# name: metrics
|
||||
# parameters:
|
||||
# - name: status
|
||||
# value: '{{tasks.train-model.status}}'
|
||||
# dependencies:
|
||||
# - train-model
|
||||
# name: notify-in-slack
|
||||
# template: slack-notify-success
|
||||
name: main
|
||||
- container:
|
||||
args:
|
||||
- |
|
||||
apt-get update && \
|
||||
apt-get install -y python3-pip git wget unzip libglib2.0-0 libsm6 libxext6 libxrender-dev && \
|
||||
pip install pillow lxml Cython contextlib2 jupyter matplotlib numpy scipy boto3 pycocotools pyyaml google-cloud-storage && \
|
||||
cd /mnt/src/tf/research && \
|
||||
export PYTHONPATH=$PYTHONPATH:` + "`pwd`:`pwd`/slim" + ` && \
|
||||
cd /mnt/src/train && \
|
||||
python convert_workflow.py \
|
||||
--extras="{{workflow.parameters.extras}}" \
|
||||
--model="{{workflow.parameters.ref-model}}" \
|
||||
--num_classes="{{workflow.parameters.sys-num-classes}}" \
|
||||
--sys_finetune_checkpoint={{workflow.parameters.sys-finetune-checkpoint}}
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
image: '{{workflow.parameters.tf-image}}'
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/data
|
||||
name: data
|
||||
- mountPath: /mnt/output
|
||||
name: output
|
||||
workingDir: /mnt/src
|
||||
nodeSelector:
|
||||
beta.kubernetes.io/instance-type: '{{workflow.parameters.sys-node-pool}}'
|
||||
inputs:
|
||||
artifacts:
|
||||
- name: data
|
||||
path: /mnt/data/datasets/
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.sys-annotation-path}}'
|
||||
- name: models
|
||||
path: /mnt/data/models/
|
||||
optional: true
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.sys-finetune-checkpoint}}'
|
||||
- git:
|
||||
repo: '{{workflow.parameters.source}}'
|
||||
revision: '{{workflow.parameters.revision}}'
|
||||
name: src
|
||||
path: /mnt/src/tf
|
||||
- git:
|
||||
repo: '{{workflow.parameters.trainingsource}}'
|
||||
revision: 'optional-artifacts'
|
||||
name: tsrc
|
||||
path: /mnt/src/train
|
||||
name: tensorflow
|
||||
outputs:
|
||||
artifacts:
|
||||
- name: model
|
||||
optional: true
|
||||
path: /mnt/output
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.sys-output-path}}'
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#- container:
|
||||
# args:
|
||||
# - SLACK_USERNAME=Onepanel SLACK_TITLE="{{workflow.name}} {{inputs.parameters.status}}"
|
||||
# SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd
|
||||
# SLACK_MESSAGE=$(cat /tmp/metrics.json)} ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
# image: technosophos/slack-notify
|
||||
# inputs:
|
||||
# artifacts:
|
||||
# - name: metrics
|
||||
# optional: true
|
||||
# path: /tmp/metrics.json
|
||||
# parameters:
|
||||
# - name: status
|
||||
# name: slack-notify-success
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
creationTimestamp: null
|
||||
name: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
- metadata:
|
||||
creationTimestamp: null
|
||||
name: output
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
@@ -0,0 +1,222 @@
|
||||
metadata:
|
||||
name: "TF Object Detection Training"
|
||||
kind: Workflow
|
||||
version: 20200824101019
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tf-object-detection-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
"used-by": "cvat"
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: source
|
||||
value: https://github.com/tensorflow/models.git
|
||||
displayName: Model source code
|
||||
type: hidden
|
||||
visibility: private
|
||||
|
||||
- name: trainingsource
|
||||
value: https://github.com/onepanelio/cvat-training.git
|
||||
type: hidden
|
||||
visibility: private
|
||||
|
||||
- name: revision
|
||||
value: v1.13.0
|
||||
type: hidden
|
||||
visibility: private
|
||||
|
||||
- name: cvat-annotation-path
|
||||
value: annotation-dump/sample_dataset
|
||||
displayName: Dataset path
|
||||
hint: Path to annotated data in default object storage (i.e S3). In CVAT, this parameter will be pre-populated.
|
||||
visibility: private
|
||||
|
||||
- name: cvat-output-path
|
||||
value: workflow-data/output/sample_output
|
||||
hint: Path to store output artifacts in default object storage (i.e s3). In CVAT, this parameter will be pre-populated.
|
||||
displayName: Workflow output path
|
||||
visibility: private
|
||||
|
||||
- name: cvat-model
|
||||
value: frcnn-res50-coco
|
||||
displayName: Model
|
||||
hint: TF Detection API's model to use for training.
|
||||
type: select.select
|
||||
visibility: public
|
||||
options:
|
||||
- name: 'Faster RCNN-ResNet 101-COCO'
|
||||
value: frcnn-res101-coco
|
||||
- name: 'Faster RCNN-ResNet 101-Low Proposal-COCO'
|
||||
value: frcnn-res101-low
|
||||
- name: 'Faster RCNN-ResNet 50-COCO'
|
||||
value: frcnn-res50-coco
|
||||
- name: 'Faster RCNN-NAS-COCO'
|
||||
value: frcnn-nas-coco
|
||||
- name: 'SSD MobileNet V1-COCO'
|
||||
value: ssd-mobilenet-v1-coco2
|
||||
- name: 'SSD MobileNet V2-COCO'
|
||||
value: ssd-mobilenet-v2-coco
|
||||
- name: 'SSDLite MobileNet-COCO'
|
||||
value: ssdlite-mobilenet-coco
|
||||
|
||||
- name: hyperparameters
|
||||
value: |-
|
||||
num-steps=10000
|
||||
displayName: Hyperparameters
|
||||
visibility: public
|
||||
type: textarea.textarea
|
||||
hint: "Please refer to our <a href='https://docs.onepanel.ai/docs/getting-started/use-cases/computervision/annotation/cvat/cvat_annotation_model#arguments-optional' target='_blank'>documentation</a> for more information on parameters. Number of classes will be automatically populated if you had 'sys-num-classes' parameter in a workflow."
|
||||
|
||||
- name: cvat-finetune-checkpoint
|
||||
value: ''
|
||||
hint: Select the last fine-tune checkpoint for this model. It may take up to 5 minutes for a recent checkpoint show here. Leave empty if this is the first time you're training this model.
|
||||
displayName: Checkpoint path
|
||||
visibility: public
|
||||
|
||||
- name: cvat-num-classes
|
||||
value: '81'
|
||||
hint: Number of classes
|
||||
displayName: Number of classes
|
||||
visibility: private
|
||||
|
||||
- name: tf-image
|
||||
value: tensorflow/tensorflow:1.13.1-py3
|
||||
type: select.select
|
||||
displayName: Select TensorFlow image
|
||||
visibility: public
|
||||
hint: Select the GPU image if you are running on a GPU node pool
|
||||
options:
|
||||
- name: 'TensorFlow 1.13.1 CPU Image'
|
||||
value: 'tensorflow/tensorflow:1.13.1-py3'
|
||||
- name: 'TensorFlow 1.13.1 GPU Image'
|
||||
value: 'tensorflow/tensorflow:1.13.1-gpu-py3'
|
||||
|
||||
- displayName: Node pool
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.select
|
||||
name: sys-node-pool
|
||||
value: Standard_D4s_v3
|
||||
visibility: public
|
||||
required: true
|
||||
options:
|
||||
- name: 'CPU: 2, RAM: 8GB'
|
||||
value: Standard_D2s_v3
|
||||
- name: 'CPU: 4, RAM: 16GB'
|
||||
value: Standard_D4s_v3
|
||||
- name: 'GPU: 1xK80, CPU: 6, RAM: 56GB'
|
||||
value: Standard_NC6
|
||||
- name: dump-format
|
||||
value: cvat_tfrecord
|
||||
visibility: public
|
||||
entrypoint: main
|
||||
templates:
|
||||
- dag:
|
||||
tasks:
|
||||
- name: train-model
|
||||
template: tensorflow
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
# - arguments:
|
||||
# artifacts:
|
||||
# - from: '{{tasks.train-model.outputs.artifacts.sys-metrics}}'
|
||||
# name: metrics
|
||||
# parameters:
|
||||
# - name: status
|
||||
# value: '{{tasks.train-model.status}}'
|
||||
# dependencies:
|
||||
# - train-model
|
||||
# name: notify-in-slack
|
||||
# template: slack-notify-success
|
||||
name: main
|
||||
- container:
|
||||
args:
|
||||
- |
|
||||
apt-get update && \
|
||||
apt-get install -y python3-pip git wget unzip libglib2.0-0 libsm6 libxext6 libxrender-dev && \
|
||||
pip install pillow lxml Cython contextlib2 jupyter matplotlib numpy scipy boto3 pycocotools pyyaml google-cloud-storage && \
|
||||
cd /mnt/src/tf/research && \
|
||||
export PYTHONPATH=$PYTHONPATH:` + "`pwd`:`pwd`" + `/slim && \
|
||||
cd /mnt/src/train && \
|
||||
python convert_workflow.py \
|
||||
--extras="{{workflow.parameters.hyperparameters}}" \
|
||||
--model="{{workflow.parameters.cvat-model}}" \
|
||||
--num_classes="{{workflow.parameters.cvat-num-classes}}" \
|
||||
--sys_finetune_checkpoint={{workflow.parameters.cvat-finetune-checkpoint}}
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
image: '{{workflow.parameters.tf-image}}'
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/data
|
||||
name: data
|
||||
- mountPath: /mnt/output
|
||||
name: output
|
||||
workingDir: /mnt/src
|
||||
nodeSelector:
|
||||
beta.kubernetes.io/instance-type: '{{workflow.parameters.sys-node-pool}}'
|
||||
inputs:
|
||||
artifacts:
|
||||
- name: data
|
||||
path: /mnt/data/datasets/
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-annotation-path}}'
|
||||
- name: models
|
||||
path: /mnt/data/models/
|
||||
optional: true
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-finetune-checkpoint}}'
|
||||
- git:
|
||||
repo: '{{workflow.parameters.source}}'
|
||||
revision: '{{workflow.parameters.revision}}'
|
||||
name: src
|
||||
path: /mnt/src/tf
|
||||
- git:
|
||||
repo: '{{workflow.parameters.trainingsource}}'
|
||||
revision: 'optional-artifacts'
|
||||
name: tsrc
|
||||
path: /mnt/src/train
|
||||
name: tensorflow
|
||||
outputs:
|
||||
artifacts:
|
||||
- name: model
|
||||
optional: true
|
||||
path: /mnt/output
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-output-path}}/{{workflow.name}}'
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#- container:
|
||||
# args:
|
||||
# - SLACK_USERNAME=Onepanel SLACK_TITLE="{{workflow.name}} {{inputs.parameters.status}}"
|
||||
# SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd
|
||||
# SLACK_MESSAGE=$(cat /tmp/metrics.json)} ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
# image: technosophos/slack-notify
|
||||
# inputs:
|
||||
# artifacts:
|
||||
# - name: metrics
|
||||
# optional: true
|
||||
# path: /tmp/metrics.json
|
||||
# parameters:
|
||||
# - name: status
|
||||
# name: slack-notify-success
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
creationTimestamp: null
|
||||
name: data
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
- metadata:
|
||||
creationTimestamp: null
|
||||
name: output
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
@@ -1,3 +1,13 @@
|
||||
metadata:
|
||||
name: "TF Object Detection Training"
|
||||
kind: Workflow
|
||||
version: 20201115134934
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tf-object-detection-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
"used-by": "cvat"
|
||||
spec:
|
||||
entrypoint: main
|
||||
arguments:
|
||||
parameters:
|
||||
@@ -158,12 +168,12 @@ templates:
|
||||
artifacts:
|
||||
- name: data
|
||||
path: /mnt/data/datasets/
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-annotation-path}}'
|
||||
- name: models
|
||||
path: /mnt/data/models/
|
||||
optional: true
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-finetune-checkpoint}}'
|
||||
- git:
|
||||
repo: '{{workflow.parameters.source}}'
|
||||
@@ -180,7 +190,7 @@ templates:
|
||||
- name: model
|
||||
optional: true
|
||||
path: /mnt/output
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-output-path}}/{{workflow.name}}'
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#- container:
|
||||
|
||||
@@ -1,3 +1,13 @@
|
||||
metadata:
|
||||
name: "TF Object Detection Training"
|
||||
kind: Workflow
|
||||
version: 20201130130433
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tf-object-detection-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
"used-by": "cvat"
|
||||
spec:
|
||||
entrypoint: main
|
||||
arguments:
|
||||
parameters:
|
||||
@@ -158,12 +168,12 @@ templates:
|
||||
artifacts:
|
||||
- name: data
|
||||
path: /mnt/data/datasets/
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-annotation-path}}'
|
||||
- name: models
|
||||
path: /mnt/data/models/
|
||||
optional: true
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.parameters.cvat-finetune-checkpoint}}'
|
||||
- git:
|
||||
repo: '{{workflow.parameters.source}}'
|
||||
@@ -180,7 +190,7 @@ templates:
|
||||
- name: model
|
||||
optional: true
|
||||
path: /mnt/output
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-output-path}}/{{workflow.name}}'
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#- container:
|
||||
|
||||
@@ -1,3 +1,13 @@
|
||||
metadata:
|
||||
name: "TF Object Detection Training"
|
||||
kind: Workflow
|
||||
version: 20201208155115
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tf-object-detection-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
"used-by": "cvat"
|
||||
spec:
|
||||
entrypoint: main
|
||||
arguments:
|
||||
parameters:
|
||||
@@ -160,12 +170,12 @@ templates:
|
||||
artifacts:
|
||||
- name: data
|
||||
path: /mnt/data/datasets/
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-annotation-path}}'
|
||||
- name: models
|
||||
path: /mnt/data/models/
|
||||
optional: true
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.parameters.cvat-finetune-checkpoint}}'
|
||||
- git:
|
||||
repo: '{{workflow.parameters.source}}'
|
||||
@@ -182,7 +192,7 @@ templates:
|
||||
- name: model
|
||||
optional: true
|
||||
path: /mnt/output
|
||||
{{.ArtifactRepositoryType}}:
|
||||
"{{.ArtifactRepositoryType}}":
|
||||
key: '{{workflow.namespace}}/{{workflow.parameters.cvat-output-path}}/{{workflow.name}}'
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#- container:
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/tf-object-detection-training/
|
||||
metadata:
|
||||
name: "TF Object Detection Training"
|
||||
kind: Workflow
|
||||
version: 20201223202929
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tf-object-detection-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
"used-by": "cvat"
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: cvat-annotation-path
|
||||
@@ -75,7 +84,7 @@ arguments:
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.nodepool
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
visibility: public
|
||||
required: true
|
||||
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
# source: https://github.com/onepanelio/templates/blob/master/workflows/tf-object-detection-training/
|
||||
metadata:
|
||||
name: "TF Object Detection Training"
|
||||
kind: Workflow
|
||||
version: 20210118175809
|
||||
action: update
|
||||
source: "https://github.com/onepanelio/templates/blob/master/workflows/tf-object-detection-training/"
|
||||
labels:
|
||||
"created-by": "system"
|
||||
"used-by": "cvat"
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
- name: cvat-annotation-path
|
||||
@@ -101,7 +110,7 @@ arguments:
|
||||
hint: Name of node pool or group to run this workflow task
|
||||
type: select.nodepool
|
||||
name: sys-node-pool
|
||||
value: {{.DefaultNodePoolOption}}
|
||||
value: "{{.DefaultNodePoolOption}}"
|
||||
visibility: public
|
||||
required: true
|
||||
|
||||
@@ -152,7 +161,7 @@ templates:
|
||||
name: output
|
||||
workingDir: /mnt/src
|
||||
nodeSelector:
|
||||
{{.NodePoolLabel}}: '{{workflow.parameters.sys-node-pool}}'
|
||||
"{{.NodePoolLabel}}": '{{workflow.parameters.sys-node-pool}}'
|
||||
inputs:
|
||||
artifacts:
|
||||
- name: data
|
||||
@@ -215,7 +224,7 @@ templates:
|
||||
name: processed-data
|
||||
workingDir: /mnt/src
|
||||
nodeSelector:
|
||||
{{.NodePoolLabel}}: '{{workflow.parameters.sys-node-pool}}'
|
||||
"{{.NodePoolLabel}}": '{{workflow.parameters.sys-node-pool}}'
|
||||
inputs:
|
||||
artifacts:
|
||||
- name: data
|
||||
|
||||
105
db/yaml/workspaces/cvat/20200528140124.yaml
Normal file
105
db/yaml/workspaces/cvat/20200528140124.yaml
Normal file
@@ -0,0 +1,105 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20200528140124
|
||||
action: create
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: root
|
||||
- name: POSTGRES_DB
|
||||
value: cvat
|
||||
- name: POSTGRES_HOST_AUTH_METHOD
|
||||
value: trust
|
||||
- name: PGDATA
|
||||
value: /var/lib/psql/data
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: tcp
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /var/lib/psql
|
||||
- name: cvat-redis
|
||||
image: redis:4.0-alpine
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: tcp
|
||||
- name: cvat
|
||||
image: onepanel/cvat:v0.7.0
|
||||
env:
|
||||
- name: DJANGO_MODWSGI_EXTRA_ARGS
|
||||
value: ""
|
||||
- name: ALLOWED_HOSTS
|
||||
value: '*'
|
||||
- name: CVAT_REDIS_HOST
|
||||
value: localhost
|
||||
- name: CVAT_POSTGRES_HOST
|
||||
value: localhost
|
||||
- name: CVAT_SHARE_URL
|
||||
value: /home/django/data
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /home/django/data
|
||||
- name: keys
|
||||
mountPath: /home/django/keys
|
||||
- name: logs
|
||||
mountPath: /home/django/logs
|
||||
- name: models
|
||||
mountPath: /home/django/models
|
||||
- name: cvat-ui
|
||||
image: onepanel/cvat-ui:v0.7.0
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
ports:
|
||||
- name: cvat-ui
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 80
|
||||
- name: cvat
|
||||
port: 8080
|
||||
protocol: TCP
|
||||
targetPort: 8080
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
regex: /api/.*|/git/.*|/tensorflow/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
|
||||
- queryParams:
|
||||
id:
|
||||
regex: \d+.*
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8080
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
# DAG Workflow to be executed once a Workspace action completes
|
||||
# postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
116
db/yaml/workspaces/cvat/20200626113635.yaml
Normal file
116
db/yaml/workspaces/cvat/20200626113635.yaml
Normal file
@@ -0,0 +1,116 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20200626113635
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: root
|
||||
- name: POSTGRES_DB
|
||||
value: cvat
|
||||
- name: POSTGRES_HOST_AUTH_METHOD
|
||||
value: trust
|
||||
- name: PGDATA
|
||||
value: /var/lib/psql/data
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: tcp
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /var/lib/psql
|
||||
- name: cvat-redis
|
||||
image: redis:4.0-alpine
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: tcp
|
||||
- name: cvat
|
||||
image: onepanel/cvat:v0.7.6
|
||||
env:
|
||||
- name: DJANGO_MODWSGI_EXTRA_ARGS
|
||||
value: ""
|
||||
- name: ALLOWED_HOSTS
|
||||
value: '*'
|
||||
- name: CVAT_REDIS_HOST
|
||||
value: localhost
|
||||
- name: CVAT_POSTGRES_HOST
|
||||
value: localhost
|
||||
- name: CVAT_SHARE_URL
|
||||
value: /home/django/data
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /home/django/data
|
||||
- name: keys
|
||||
mountPath: /home/django/keys
|
||||
- name: logs
|
||||
mountPath: /home/django/logs
|
||||
- name: models
|
||||
mountPath: /home/django/models
|
||||
- name: share
|
||||
mountPath: /home/django/share
|
||||
- name: cvat-ui
|
||||
image: onepanel/cvat-ui:v0.7.5
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
- name: filesyncer
|
||||
image: onepanel/filesyncer:v0.0.4
|
||||
command: ['python3', 'main.py']
|
||||
volumeMounts:
|
||||
- name: share
|
||||
mountPath: /mnt/share
|
||||
ports:
|
||||
- name: cvat-ui
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 80
|
||||
- name: cvat
|
||||
port: 8080
|
||||
protocol: TCP
|
||||
targetPort: 8080
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
regex: /api/.*|/git/.*|/tensorflow/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
|
||||
- queryParams:
|
||||
id:
|
||||
regex: \d+.*
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8080
|
||||
timeout: 600s
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
timeout: 600s
|
||||
# DAG Workflow to be executed once a Workspace action completes (optional)
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
118
db/yaml/workspaces/cvat/20200704151301.yaml
Normal file
118
db/yaml/workspaces/cvat/20200704151301.yaml
Normal file
@@ -0,0 +1,118 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20200704151301
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: root
|
||||
- name: POSTGRES_DB
|
||||
value: cvat
|
||||
- name: POSTGRES_HOST_AUTH_METHOD
|
||||
value: trust
|
||||
- name: PGDATA
|
||||
value: /var/lib/psql/data
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: tcp
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /var/lib/psql
|
||||
- name: cvat-redis
|
||||
image: redis:4.0-alpine
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: tcp
|
||||
- name: cvat
|
||||
image: onepanel/cvat:v0.7.10-stable
|
||||
env:
|
||||
- name: DJANGO_MODWSGI_EXTRA_ARGS
|
||||
value: ""
|
||||
- name: ALLOWED_HOSTS
|
||||
value: '*'
|
||||
- name: CVAT_REDIS_HOST
|
||||
value: localhost
|
||||
- name: CVAT_POSTGRES_HOST
|
||||
value: localhost
|
||||
- name: CVAT_SHARE_URL
|
||||
value: /home/django/data
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /home/django/data
|
||||
- name: keys
|
||||
mountPath: /home/django/keys
|
||||
- name: logs
|
||||
mountPath: /home/django/logs
|
||||
- name: models
|
||||
mountPath: /home/django/models
|
||||
- name: share
|
||||
mountPath: /home/django/share
|
||||
- name: cvat-ui
|
||||
image: onepanel/cvat-ui:v0.7.10-stable
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
# Uncomment following lines to enable S3 FileSyncer
|
||||
# Refer to https://docs.onepanel.ai/docs/getting-started/use-cases/computervision/annotation/cvat/cvat_quick_guide#setting-up-environment-variables
|
||||
#- name: filesyncer
|
||||
# image: onepanel/filesyncer:v0.0.4
|
||||
# command: ['python3', 'main.py']
|
||||
# volumeMounts:
|
||||
# - name: share
|
||||
# mountPath: /mnt/share
|
||||
ports:
|
||||
- name: cvat-ui
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 80
|
||||
- name: cvat
|
||||
port: 8080
|
||||
protocol: TCP
|
||||
targetPort: 8080
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
regex: /api/.*|/git/.*|/tensorflow/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
|
||||
- queryParams:
|
||||
id:
|
||||
regex: \d+.*
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8080
|
||||
timeout: 600s
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
timeout: 600s
|
||||
# DAG Workflow to be executed once a Workspace action completes (optional)
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
135
db/yaml/workspaces/cvat/20200724220450.yaml
Normal file
135
db/yaml/workspaces/cvat/20200724220450.yaml
Normal file
@@ -0,0 +1,135 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20200724220450
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
- name: storage-prefix
|
||||
displayName: Directory in default object storage
|
||||
value: data
|
||||
hint: Location of data and models in default object storage, will continuously sync to '/mnt/share'
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: root
|
||||
- name: POSTGRES_DB
|
||||
value: cvat
|
||||
- name: POSTGRES_HOST_AUTH_METHOD
|
||||
value: trust
|
||||
- name: PGDATA
|
||||
value: /var/lib/psql/data
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: tcp
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /var/lib/psql
|
||||
- name: cvat-redis
|
||||
image: redis:4.0-alpine
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: tcp
|
||||
- name: cvat
|
||||
image: onepanel/cvat:v0.7.10-stable
|
||||
env:
|
||||
- name: DJANGO_MODWSGI_EXTRA_ARGS
|
||||
value: ""
|
||||
- name: ALLOWED_HOSTS
|
||||
value: '*'
|
||||
- name: CVAT_REDIS_HOST
|
||||
value: localhost
|
||||
- name: CVAT_POSTGRES_HOST
|
||||
value: localhost
|
||||
- name: CVAT_SHARE_URL
|
||||
value: /home/django/data
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /home/django/data
|
||||
- name: keys
|
||||
mountPath: /home/django/keys
|
||||
- name: logs
|
||||
mountPath: /home/django/logs
|
||||
- name: models
|
||||
mountPath: /home/django/models
|
||||
- name: share
|
||||
mountPath: /home/django/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
- name: cvat-ui
|
||||
image: onepanel/cvat-ui:v0.7.10-stable
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
# You can add multiple FileSyncer sidecar containers if needed
|
||||
- name: filesyncer
|
||||
image: "onepanel/filesyncer:{{.ArtifactRepositoryType}}"
|
||||
args:
|
||||
- download
|
||||
env:
|
||||
- name: FS_PATH
|
||||
value: /mnt/share
|
||||
- name: FS_PREFIX
|
||||
value: '{{workspace.parameters.storage-prefix}}'
|
||||
volumeMounts:
|
||||
- name: share
|
||||
mountPath: /mnt/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
ports:
|
||||
- name: cvat-ui
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 80
|
||||
- name: cvat
|
||||
port: 8080
|
||||
protocol: TCP
|
||||
targetPort: 8080
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
regex: /api/.*|/git/.*|/tensorflow/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
|
||||
- queryParams:
|
||||
id:
|
||||
regex: \d+.*
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8080
|
||||
timeout: 600s
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
timeout: 600s
|
||||
# DAG Workflow to be executed once a Workspace action completes (optional)
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
144
db/yaml/workspaces/cvat/20200812113316.yaml
Normal file
144
db/yaml/workspaces/cvat/20200812113316.yaml
Normal file
@@ -0,0 +1,144 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20200812113316
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sync-directory
|
||||
displayName: Directory to sync raw input and training output
|
||||
value: workflow-data
|
||||
hint: Location to sync raw input, models and checkpoints from default object storage. Note that this will be relative to the current namespace.
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: root
|
||||
- name: POSTGRES_DB
|
||||
value: cvat
|
||||
- name: POSTGRES_HOST_AUTH_METHOD
|
||||
value: trust
|
||||
- name: PGDATA
|
||||
value: /var/lib/psql/data
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: tcp
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /var/lib/psql
|
||||
- name: cvat-redis
|
||||
image: redis:4.0-alpine
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: tcp
|
||||
- name: cvat
|
||||
image: onepanel/cvat:0.12.0_cvat.1.0.0-beta.2-cuda
|
||||
env:
|
||||
- name: DJANGO_MODWSGI_EXTRA_ARGS
|
||||
value: ""
|
||||
- name: ALLOWED_HOSTS
|
||||
value: '*'
|
||||
- name: CVAT_REDIS_HOST
|
||||
value: localhost
|
||||
- name: CVAT_POSTGRES_HOST
|
||||
value: localhost
|
||||
- name: CVAT_SHARE_URL
|
||||
value: /home/django/data
|
||||
- name: ONEPANEL_SYNC_DIRECTORY
|
||||
value: '{{workspace.parameters.sync-directory}}'
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: all
|
||||
- name: NVIDIA_DRIVER_CAPABILITIES
|
||||
value: compute,utility
|
||||
- name: NVIDIA_REQUIRE_CUDA
|
||||
value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /home/django/data
|
||||
- name: keys
|
||||
mountPath: /home/django/keys
|
||||
- name: logs
|
||||
mountPath: /home/django/logs
|
||||
- name: models
|
||||
mountPath: /home/django/models
|
||||
- name: share
|
||||
mountPath: /home/django/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
- name: cvat-ui
|
||||
image: onepanel/cvat-ui:0.12.0_cvat.1.0.0-beta.2
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
# You can add multiple FileSyncer sidecar containers if needed
|
||||
- name: filesyncer
|
||||
image: "onepanel/filesyncer:{{.ArtifactRepositoryType}}"
|
||||
imagePullPolicy: Always
|
||||
args:
|
||||
- download
|
||||
env:
|
||||
- name: FS_PATH
|
||||
value: /mnt/share
|
||||
- name: FS_PREFIX
|
||||
value: '{{workflow.namespace}}/{{workspace.parameters.sync-directory}}'
|
||||
volumeMounts:
|
||||
- name: share
|
||||
mountPath: /mnt/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
ports:
|
||||
- name: cvat-ui
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 80
|
||||
- name: cvat
|
||||
port: 8080
|
||||
protocol: TCP
|
||||
targetPort: 8080
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
|
||||
- queryParams:
|
||||
id:
|
||||
regex: \d+.*
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8080
|
||||
timeout: 600s
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
timeout: 600s
|
||||
# DAG Workflow to be executed once a Workspace action completes (optional)
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
144
db/yaml/workspaces/cvat/20200824101905.yaml
Normal file
144
db/yaml/workspaces/cvat/20200824101905.yaml
Normal file
@@ -0,0 +1,144 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20200824101905
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sync-directory
|
||||
displayName: Directory to sync raw input and training output
|
||||
value: workflow-data
|
||||
hint: Location to sync raw input, models and checkpoints from default object storage. Note that this will be relative to the current namespace.
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: root
|
||||
- name: POSTGRES_DB
|
||||
value: cvat
|
||||
- name: POSTGRES_HOST_AUTH_METHOD
|
||||
value: trust
|
||||
- name: PGDATA
|
||||
value: /var/lib/psql/data
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: tcp
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /var/lib/psql
|
||||
- name: cvat-redis
|
||||
image: redis:4.0-alpine
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: tcp
|
||||
- name: cvat
|
||||
image: onepanel/cvat:0.12.0-rc.6_cvat.1.0.0
|
||||
env:
|
||||
- name: DJANGO_MODWSGI_EXTRA_ARGS
|
||||
value: ""
|
||||
- name: ALLOWED_HOSTS
|
||||
value: '*'
|
||||
- name: CVAT_REDIS_HOST
|
||||
value: localhost
|
||||
- name: CVAT_POSTGRES_HOST
|
||||
value: localhost
|
||||
- name: CVAT_SHARE_URL
|
||||
value: /home/django/data
|
||||
- name: ONEPANEL_SYNC_DIRECTORY
|
||||
value: '{{workspace.parameters.sync-directory}}'
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: all
|
||||
- name: NVIDIA_DRIVER_CAPABILITIES
|
||||
value: compute,utility
|
||||
- name: NVIDIA_REQUIRE_CUDA
|
||||
value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /home/django/data
|
||||
- name: keys
|
||||
mountPath: /home/django/keys
|
||||
- name: logs
|
||||
mountPath: /home/django/logs
|
||||
- name: models
|
||||
mountPath: /home/django/models
|
||||
- name: share
|
||||
mountPath: /home/django/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
- name: cvat-ui
|
||||
image: onepanel/cvat-ui:0.12.0-rc.1_cvat.1.0.0
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
# You can add multiple FileSyncer sidecar containers if needed
|
||||
- name: filesyncer
|
||||
image: "onepanel/filesyncer:{{.ArtifactRepositoryType}}"
|
||||
imagePullPolicy: Always
|
||||
args:
|
||||
- download
|
||||
env:
|
||||
- name: FS_PATH
|
||||
value: /mnt/share
|
||||
- name: FS_PREFIX
|
||||
value: '{{workflow.namespace}}/{{workspace.parameters.sync-directory}}'
|
||||
volumeMounts:
|
||||
- name: share
|
||||
mountPath: /mnt/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
ports:
|
||||
- name: cvat-ui
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 80
|
||||
- name: cvat
|
||||
port: 8080
|
||||
protocol: TCP
|
||||
targetPort: 8080
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
|
||||
- queryParams:
|
||||
id:
|
||||
regex: \d+.*
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8080
|
||||
timeout: 600s
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
timeout: 600s
|
||||
# DAG Workflow to be executed once a Workspace action completes (optional)
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
144
db/yaml/workspaces/cvat/20200825154403.yaml
Normal file
144
db/yaml/workspaces/cvat/20200825154403.yaml
Normal file
@@ -0,0 +1,144 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20200825154403
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sync-directory
|
||||
displayName: Directory to sync raw input and training output
|
||||
value: workflow-data
|
||||
hint: Location to sync raw input, models and checkpoints from default object storage. Note that this will be relative to the current namespace.
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: root
|
||||
- name: POSTGRES_DB
|
||||
value: cvat
|
||||
- name: POSTGRES_HOST_AUTH_METHOD
|
||||
value: trust
|
||||
- name: PGDATA
|
||||
value: /var/lib/psql/data
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: tcp
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /var/lib/psql
|
||||
- name: cvat-redis
|
||||
image: redis:4.0-alpine
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: tcp
|
||||
- name: cvat
|
||||
image: onepanel/cvat:0.12.0_cvat.1.0.0
|
||||
env:
|
||||
- name: DJANGO_MODWSGI_EXTRA_ARGS
|
||||
value: ""
|
||||
- name: ALLOWED_HOSTS
|
||||
value: '*'
|
||||
- name: CVAT_REDIS_HOST
|
||||
value: localhost
|
||||
- name: CVAT_POSTGRES_HOST
|
||||
value: localhost
|
||||
- name: CVAT_SHARE_URL
|
||||
value: /home/django/data
|
||||
- name: ONEPANEL_SYNC_DIRECTORY
|
||||
value: '{{workspace.parameters.sync-directory}}'
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: all
|
||||
- name: NVIDIA_DRIVER_CAPABILITIES
|
||||
value: compute,utility
|
||||
- name: NVIDIA_REQUIRE_CUDA
|
||||
value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /home/django/data
|
||||
- name: keys
|
||||
mountPath: /home/django/keys
|
||||
- name: logs
|
||||
mountPath: /home/django/logs
|
||||
- name: models
|
||||
mountPath: /home/django/models
|
||||
- name: share
|
||||
mountPath: /home/django/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
- name: cvat-ui
|
||||
image: onepanel/cvat-ui:0.12.0_cvat.1.0.0
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
# You can add multiple FileSyncer sidecar containers if needed
|
||||
- name: filesyncer
|
||||
image: "onepanel/filesyncer:{{.ArtifactRepositoryType}}"
|
||||
imagePullPolicy: Always
|
||||
args:
|
||||
- download
|
||||
env:
|
||||
- name: FS_PATH
|
||||
value: /mnt/share
|
||||
- name: FS_PREFIX
|
||||
value: '{{workflow.namespace}}/{{workspace.parameters.sync-directory}}'
|
||||
volumeMounts:
|
||||
- name: share
|
||||
mountPath: /mnt/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
ports:
|
||||
- name: cvat-ui
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 80
|
||||
- name: cvat
|
||||
port: 8080
|
||||
protocol: TCP
|
||||
targetPort: 8080
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
|
||||
- queryParams:
|
||||
id:
|
||||
regex: \d+.*
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8080
|
||||
timeout: 600s
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
timeout: 600s
|
||||
# DAG Workflow to be executed once a Workspace action completes (optional)
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
156
db/yaml/workspaces/cvat/20200826185926.yaml
Normal file
156
db/yaml/workspaces/cvat/20200826185926.yaml
Normal file
@@ -0,0 +1,156 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20200826185926
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sync-directory
|
||||
displayName: Directory to sync raw input and training output
|
||||
value: workflow-data
|
||||
hint: Location to sync raw input, models and checkpoints from default object storage. Note that this will be relative to the current namespace.
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: root
|
||||
- name: POSTGRES_DB
|
||||
value: cvat
|
||||
- name: POSTGRES_HOST_AUTH_METHOD
|
||||
value: trust
|
||||
- name: PGDATA
|
||||
value: /var/lib/psql/data
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: tcp
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /var/lib/psql
|
||||
- name: cvat-redis
|
||||
image: redis:4.0-alpine
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: tcp
|
||||
- name: cvat
|
||||
image: onepanel/cvat:0.12.0_cvat.1.0.0
|
||||
env:
|
||||
- name: DJANGO_MODWSGI_EXTRA_ARGS
|
||||
value: ""
|
||||
- name: ALLOWED_HOSTS
|
||||
value: '*'
|
||||
- name: CVAT_REDIS_HOST
|
||||
value: localhost
|
||||
- name: CVAT_POSTGRES_HOST
|
||||
value: localhost
|
||||
- name: CVAT_SHARE_URL
|
||||
value: /home/django/data
|
||||
- name: ONEPANEL_SYNC_DIRECTORY
|
||||
value: '{{workspace.parameters.sync-directory}}'
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: all
|
||||
- name: NVIDIA_DRIVER_CAPABILITIES
|
||||
value: compute,utility
|
||||
- name: NVIDIA_REQUIRE_CUDA
|
||||
value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /home/django/data
|
||||
- name: keys
|
||||
mountPath: /home/django/keys
|
||||
- name: logs
|
||||
mountPath: /home/django/logs
|
||||
- name: models
|
||||
mountPath: /home/django/models
|
||||
- name: share
|
||||
mountPath: /home/django/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
- name: cvat-ui
|
||||
image: onepanel/cvat-ui:0.12.0_cvat.1.0.0
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
# You can add multiple FileSyncer sidecar containers if needed
|
||||
- name: filesyncer
|
||||
image: "onepanel/filesyncer:{{.ArtifactRepositoryType}}"
|
||||
imagePullPolicy: Always
|
||||
args:
|
||||
- download
|
||||
- -server-prefix=/sys/filesyncer
|
||||
env:
|
||||
- name: FS_PATH
|
||||
value: /mnt/share
|
||||
- name: FS_PREFIX
|
||||
value: '{{workflow.namespace}}/{{workspace.parameters.sync-directory}}'
|
||||
volumeMounts:
|
||||
- name: share
|
||||
mountPath: /mnt/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
ports:
|
||||
- name: cvat-ui
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 80
|
||||
- name: cvat
|
||||
port: 8080
|
||||
protocol: TCP
|
||||
targetPort: 8080
|
||||
- name: fs
|
||||
port: 8888
|
||||
protocol: TCP
|
||||
targetPort: 8888
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /sys/filesyncer
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8888
|
||||
- match:
|
||||
- uri:
|
||||
regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
|
||||
- queryParams:
|
||||
id:
|
||||
regex: \d+.*
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8080
|
||||
timeout: 600s
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
timeout: 600s
|
||||
# DAG Workflow to be executed once a Workspace action completes (optional)
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
154
db/yaml/workspaces/cvat/20201001070806.yaml
Normal file
154
db/yaml/workspaces/cvat/20201001070806.yaml
Normal file
@@ -0,0 +1,154 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20201001070806
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sync-directory
|
||||
displayName: Directory to sync raw input and training output
|
||||
value: workflow-data
|
||||
hint: Location to sync raw input, models and checkpoints from default object storage. Note that this will be relative to the current namespace.
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: root
|
||||
- name: POSTGRES_DB
|
||||
value: cvat
|
||||
- name: POSTGRES_HOST_AUTH_METHOD
|
||||
value: trust
|
||||
- name: PGDATA
|
||||
value: /var/lib/psql/data
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: tcp
|
||||
volumeMounts:
|
||||
- name: db
|
||||
mountPath: /var/lib/psql
|
||||
- name: cvat-redis
|
||||
image: redis:4.0-alpine
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
name: tcp
|
||||
- name: cvat
|
||||
image: onepanel/cvat:0.12.1_cvat.1.0.0
|
||||
env:
|
||||
- name: DJANGO_MODWSGI_EXTRA_ARGS
|
||||
value: ""
|
||||
- name: ALLOWED_HOSTS
|
||||
value: '*'
|
||||
- name: CVAT_REDIS_HOST
|
||||
value: localhost
|
||||
- name: CVAT_POSTGRES_HOST
|
||||
value: localhost
|
||||
- name: CVAT_SHARE_URL
|
||||
value: /home/django/data
|
||||
- name: ONEPANEL_SYNC_DIRECTORY
|
||||
value: '{{workspace.parameters.sync-directory}}'
|
||||
- name: NVIDIA_VISIBLE_DEVICES
|
||||
value: all
|
||||
- name: NVIDIA_DRIVER_CAPABILITIES
|
||||
value: compute,utility
|
||||
- name: NVIDIA_REQUIRE_CUDA
|
||||
value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /home/django/data
|
||||
- name: keys
|
||||
mountPath: /home/django/keys
|
||||
- name: logs
|
||||
mountPath: /home/django/logs
|
||||
- name: models
|
||||
mountPath: /home/django/models
|
||||
- name: share
|
||||
mountPath: /home/django/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
- name: cvat-ui
|
||||
image: onepanel/cvat-ui:0.12.1_cvat.1.0.0
|
||||
ports:
|
||||
- containerPort: 80
|
||||
name: http
|
||||
# You can add multiple FileSyncer sidecar containers if needed
|
||||
- name: filesyncer
|
||||
image: "onepanel/filesyncer:{{.ArtifactRepositoryType}}"
|
||||
imagePullPolicy: Always
|
||||
args:
|
||||
- download
|
||||
- -server-prefix=/sys/filesyncer
|
||||
env:
|
||||
- name: FS_PATH
|
||||
value: /mnt/share
|
||||
- name: FS_PREFIX
|
||||
value: '{{workflow.namespace}}/{{workspace.parameters.sync-directory}}'
|
||||
volumeMounts:
|
||||
- name: share
|
||||
mountPath: /mnt/share
|
||||
- name: sys-namespace-config
|
||||
mountPath: /etc/onepanel
|
||||
readOnly: true
|
||||
ports:
|
||||
- name: cvat-ui
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 80
|
||||
- name: cvat
|
||||
port: 8080
|
||||
protocol: TCP
|
||||
targetPort: 8080
|
||||
- name: fs
|
||||
port: 8888
|
||||
protocol: TCP
|
||||
targetPort: 8888
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /sys/filesyncer
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8888
|
||||
- match:
|
||||
- uri:
|
||||
regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
|
||||
- queryParams:
|
||||
id:
|
||||
regex: \d+.*
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 8080
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
# DAG Workflow to be executed once a Workspace action completes (optional)
|
||||
# Uncomment the lines below if you want to send Slack notifications
|
||||
#postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20201016170415
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
@@ -73,7 +80,7 @@ containers:
|
||||
name: http
|
||||
# You can add multiple FileSyncer sidecar containers if needed
|
||||
- name: filesyncer
|
||||
image: onepanel/filesyncer:{{.ArtifactRepositoryType}}
|
||||
image: "onepanel/filesyncer:{{.ArtifactRepositoryType}}"
|
||||
imagePullPolicy: Always
|
||||
args:
|
||||
- download
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20201102104048
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20201113094916
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20201115133046
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20201211161117
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20210107094725
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
# Workspace arguments
|
||||
arguments:
|
||||
parameters:
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20210129134326
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20210224180017
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20210323175655
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: CVAT
|
||||
kind: Workspace
|
||||
version: 20210719190719
|
||||
action: update
|
||||
description: "Powerful and efficient Computer Vision Annotation Tool (CVAT)"
|
||||
spec:
|
||||
containers:
|
||||
- name: cvat-db
|
||||
image: postgres:10-alpine
|
||||
|
||||
64
db/yaml/workspaces/jupyterlab/20200525160514.yaml
Normal file
64
db/yaml/workspaces/jupyterlab/20200525160514.yaml
Normal file
@@ -0,0 +1,64 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20200525160514
|
||||
action: create
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: jupyterlab-tensorflow
|
||||
image: jupyter/tensorflow-notebook
|
||||
command: [start.sh, jupyter]
|
||||
env:
|
||||
- name: tornado
|
||||
value: "{ 'headers': { 'Content-Security-Policy': \"frame-ancestors * 'self'\" } }"
|
||||
args:
|
||||
- lab
|
||||
- --LabApp.token=''
|
||||
- --LabApp.allow_remote_access=True
|
||||
- --LabApp.allow_origin="*"
|
||||
- --LabApp.disable_check_xsrf=True
|
||||
- --LabApp.trust_xheaders=True
|
||||
- --LabApp.tornado_settings=$(tornado)
|
||||
- --notebook-dir='/data'
|
||||
ports:
|
||||
- containerPort: 8888
|
||||
name: jupyterlab
|
||||
# Volumes to be mounted in this container
|
||||
# Onepanel will automatically create these volumes and mount them to the container
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
# Ports that need to be exposed
|
||||
ports:
|
||||
- name: jupyterlab
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 8888
|
||||
# Routes that will map to ports
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
# DAG Workflow to be executed once a Workspace action completes
|
||||
# postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
65
db/yaml/workspaces/jupyterlab/20200821162630.yaml
Normal file
65
db/yaml/workspaces/jupyterlab/20200821162630.yaml
Normal file
@@ -0,0 +1,65 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20200821162630
|
||||
action: update
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: jupyterlab-tensorflow
|
||||
image: onepanel/jupyterlab:1.0.1
|
||||
command: ["/bin/bash", "-c", "start.sh jupyter lab --LabApp.token='' --LabApp.allow_remote_access=True --LabApp.allow_origin=\"*\" --LabApp.disable_check_xsrf=True --LabApp.trust_xheaders=True --LabApp.base_url=/ --LabApp.tornado_settings='{\"headers\":{\"Content-Security-Policy\":\"frame-ancestors * \'self\'\"}}' --notebook-dir='/data' --allow-root"]
|
||||
env:
|
||||
- name: tornado
|
||||
value: "'{'headers':{'Content-Security-Policy':\"frame-ancestors\ *\ \'self'\"}}'"
|
||||
args:
|
||||
ports:
|
||||
- containerPort: 8888
|
||||
name: jupyterlab
|
||||
- containerPort: 6006
|
||||
name: tensorboard
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
ports:
|
||||
- name: jupyterlab
|
||||
port: 80
|
||||
protocol: TCP
|
||||
targetPort: 8888
|
||||
- name: tensorboard
|
||||
port: 6006
|
||||
protocol: TCP
|
||||
targetPort: 6006
|
||||
routes:
|
||||
- match:
|
||||
- uri:
|
||||
prefix: /tensorboard
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 6006
|
||||
- match:
|
||||
- uri:
|
||||
prefix: / #jupyter runs at the default route
|
||||
route:
|
||||
- destination:
|
||||
port:
|
||||
number: 80
|
||||
# DAG Workflow to be executed once a Workspace action completes (optional)
|
||||
#postExecutionWorkflow:
|
||||
# entrypoint: main
|
||||
# templates:
|
||||
# - name: main
|
||||
# dag:
|
||||
# tasks:
|
||||
# - name: slack-notify
|
||||
# template: slack-notify
|
||||
# - name: slack-notify
|
||||
# container:
|
||||
# image: technosophos/slack-notify
|
||||
# args:
|
||||
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
|
||||
# command:
|
||||
# - sh
|
||||
# - -c
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20200929153931
|
||||
action: update
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: jupyterlab-tensorflow
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20201028145442
|
||||
action: update
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: jupyterlab
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20201031165106
|
||||
action: update
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: jupyterlab
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20201214133458
|
||||
action: update
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: jupyterlab
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20201229205644
|
||||
action: update
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: jupyterlab
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20210129142057
|
||||
action: update
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
containers:
|
||||
- name: jupyterlab
|
||||
image: onepanel/dl:0.17.0
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20210224180017
|
||||
action: update
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
containers:
|
||||
- name: jupyterlab
|
||||
image: onepanel/dl:0.17.0
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20210323175655
|
||||
action: update
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
containers:
|
||||
- name: jupyterlab
|
||||
image: onepanel/dl:v0.20.0
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: JupyterLab
|
||||
kind: Workspace
|
||||
version: 20210719190719
|
||||
action: update
|
||||
description: "Interactive development environment for notebooks"
|
||||
spec:
|
||||
containers:
|
||||
- name: jupyterlab
|
||||
image: onepanel/dl:v0.20.0
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: "Deep Learning Desktop"
|
||||
kind: Workspace
|
||||
version: 20210414165510
|
||||
action: create
|
||||
description: "Deep learning desktop with VNC"
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
# parameter screen-resolution allows users to select screen resolution
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: "Deep Learning Desktop"
|
||||
kind: Workspace
|
||||
version: 20210719190719
|
||||
action: update
|
||||
description: "Deep learning desktop with VNC"
|
||||
spec:
|
||||
arguments:
|
||||
parameters:
|
||||
# parameter screen-resolution allows users to select screen resolution
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
metadata:
|
||||
name: "Visual Studio Code"
|
||||
kind: Workspace
|
||||
version: 20200929144301
|
||||
action: create
|
||||
description: "Open source code editor"
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: vscode
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
metadata:
|
||||
name: "Visual Studio Code"
|
||||
kind: Workspace
|
||||
version: 20201028145443
|
||||
action: update
|
||||
spec:
|
||||
# Docker containers that are part of the Workspace
|
||||
containers:
|
||||
- name: vscode
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
metadata:
|
||||
name: "Visual Studio Code"
|
||||
kind: Workspace
|
||||
version: 20210129152427
|
||||
action: update
|
||||
spec:
|
||||
containers:
|
||||
- name: vscode
|
||||
image: onepanel/vscode:1.0.0
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
metadata:
|
||||
name: "Visual Studio Code"
|
||||
kind: Workspace
|
||||
version: 20210224180017
|
||||
action: update
|
||||
spec:
|
||||
containers:
|
||||
- name: vscode
|
||||
image: onepanel/vscode:1.0.0
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
metadata:
|
||||
name: "Visual Studio Code"
|
||||
kind: Workspace
|
||||
version: 20210323175655
|
||||
action: update
|
||||
spec:
|
||||
containers:
|
||||
- name: vscode
|
||||
image: onepanel/vscode:v0.20.0_code-server.3.9.1
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
metadata:
|
||||
name: "Visual Studio Code"
|
||||
kind: Workspace
|
||||
version: 20210719190719
|
||||
action: update
|
||||
spec:
|
||||
containers:
|
||||
- name: vscode
|
||||
image: onepanel/vscode:v0.20.0_code-server.3.9.1
|
||||
|
||||
135
pkg/data.go
Normal file
135
pkg/data.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"github.com/onepanelio/core/pkg/util/data"
|
||||
"github.com/onepanelio/core/pkg/util/extensions"
|
||||
)
|
||||
|
||||
// createWorkspaceTemplateFromGenericFile will create the workspace template given by {{templateName}} with the contents
|
||||
// given by {{filename}} for the input {{namespace}}
|
||||
func (c *Client) createWorkspaceTemplateFromGenericManifest(namespace string, manifestFile *data.ManifestFile) (err error) {
|
||||
manifest, err := manifestFile.SpecString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
templateName := manifestFile.Metadata.Name
|
||||
description := manifestFile.Metadata.Description
|
||||
|
||||
artifactRepositoryType, err := c.GetArtifactRepositoryType(namespace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
replaceMap := map[string]string{
|
||||
"{{.ArtifactRepositoryType}}": artifactRepositoryType,
|
||||
}
|
||||
manifest = extensions.ReplaceMapValues(manifest, replaceMap)
|
||||
|
||||
workspaceTemplate, err := CreateWorkspaceTemplate(templateName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
workspaceTemplate.Manifest = manifest
|
||||
|
||||
if description != nil {
|
||||
workspaceTemplate.Description = *description
|
||||
}
|
||||
|
||||
_, err = c.CreateWorkspaceTemplate(namespace, workspaceTemplate)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// updateWorkspaceTemplateManifest will update the workspace template given by {{templateName}} with the contents
|
||||
// given by {{filename}}
|
||||
func (c *Client) updateWorkspaceTemplateManifest(namespace string, manifestFile *data.ManifestFile) (err error) {
|
||||
manifest, err := manifestFile.SpecString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
templateName := manifestFile.Metadata.Name
|
||||
|
||||
artifactRepositoryType, err := c.GetArtifactRepositoryType(namespace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
replaceMap := map[string]string{
|
||||
"{{.ArtifactRepositoryType}}": artifactRepositoryType,
|
||||
}
|
||||
manifest = extensions.ReplaceMapValues(manifest, replaceMap)
|
||||
|
||||
workspaceTemplate, err := CreateWorkspaceTemplate(templateName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
workspaceTemplate.Manifest = manifest
|
||||
|
||||
_, err = c.UpdateWorkspaceTemplateManifest(namespace, workspaceTemplate.UID, workspaceTemplate.Manifest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// createWorkflowTemplate will create the workflow template given by {{templateName}} with the contents
|
||||
// given by {{filename}}
|
||||
func (c *Client) createWorkflowTemplateFromGenericManifest(namespace string, manifestFile *data.ManifestFile) (err error) {
|
||||
manifest, err := manifestFile.SpecString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
templateName := manifestFile.Metadata.Name
|
||||
labels := manifestFile.Metadata.Labels
|
||||
|
||||
artifactRepositoryType, err := c.GetArtifactRepositoryType(namespace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
replaceMap := map[string]string{
|
||||
"{{.ArtifactRepositoryType}}": artifactRepositoryType,
|
||||
}
|
||||
manifest = extensions.ReplaceMapValues(manifest, replaceMap)
|
||||
|
||||
workflowTemplate, err := CreateWorkflowTemplate(templateName)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
workflowTemplate.Manifest = manifest
|
||||
workflowTemplate.Labels = labels
|
||||
|
||||
_, err = c.CreateWorkflowTemplate(namespace, workflowTemplate)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// updateWorkflowTemplateManifest will update the workflow template given by {{templateName}} with the contents
|
||||
// given by {{filename}}
|
||||
func (c *Client) updateWorkflowTemplateManifest(namespace string, manifestFile *data.ManifestFile) (err error) {
|
||||
manifest, err := manifestFile.SpecString()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
templateName := manifestFile.Metadata.Name
|
||||
labels := manifestFile.Metadata.Labels
|
||||
|
||||
artifactRepositoryType, err := c.GetArtifactRepositoryType(namespace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
replaceMap := map[string]string{
|
||||
"{{.ArtifactRepositoryType}}": artifactRepositoryType,
|
||||
}
|
||||
manifest = extensions.ReplaceMapValues(manifest, replaceMap)
|
||||
|
||||
workflowTemplate, err := CreateWorkflowTemplate(templateName)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
workflowTemplate.Manifest = manifest
|
||||
workflowTemplate.Labels = labels
|
||||
|
||||
_, err = c.CreateWorkflowTemplateVersion(namespace, workflowTemplate)
|
||||
|
||||
return
|
||||
}
|
||||
49
pkg/util/data/migration.go
Normal file
49
pkg/util/data/migration.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"gopkg.in/yaml.v3"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
// ManifestFile represents a file that contains information about a workflow or workspace template
|
||||
type ManifestFile struct {
|
||||
Metadata ManifestFileMetadata `yaml:"metadata"`
|
||||
Spec interface{} `yaml:"spec"`
|
||||
}
|
||||
|
||||
// ManifestFileMetadata represents information about the tempalte we are working with
|
||||
type ManifestFileMetadata struct {
|
||||
Name string
|
||||
Kind string // {Workflow, Workspace}
|
||||
Version uint64
|
||||
Action string // {create,update}
|
||||
Description *string
|
||||
Labels map[string]string
|
||||
Deprecated *bool
|
||||
Source *string
|
||||
}
|
||||
|
||||
// SpecString returns the spec of a manifest file as a string
|
||||
func (m *ManifestFile) SpecString() (string, error) {
|
||||
data, err := yaml.Marshal(m.Spec)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
// ManifestFileFromFile loads a manifest from a yaml file.
|
||||
func ManifestFileFromFile(path string) (*ManifestFile, error) {
|
||||
fileData, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
manifest := &ManifestFile{}
|
||||
if err := yaml.Unmarshal(fileData, manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return manifest, nil
|
||||
}
|
||||
@@ -213,3 +213,17 @@ func DeleteNode(node *yaml.Node, key *YamlIndex) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReplaceMapValues will replace strings that are keys in the input map with their values
|
||||
// the result is returned
|
||||
func ReplaceMapValues(value string, replaceMap map[string]string) string {
|
||||
replacePairs := make([]string, 0)
|
||||
|
||||
for key, value := range replaceMap {
|
||||
replacePairs = append(replacePairs, key)
|
||||
replacePairs = append(replacePairs, value)
|
||||
}
|
||||
|
||||
return strings.NewReplacer(replacePairs...).
|
||||
Replace(value)
|
||||
}
|
||||
|
||||
@@ -55,6 +55,22 @@ func (wt *WorkflowTemplate) GenerateUID(name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateWorkflowTemplate creates a new workflow template with the given name.
|
||||
// All fields that can be generated in memory without external requests are filled out, such as the UID.
|
||||
func CreateWorkflowTemplate(name string) (*WorkflowTemplate, error) {
|
||||
nameUID, err := uid2.GenerateUID(name, 30)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
workflowTemplate := &WorkflowTemplate{
|
||||
Name: name,
|
||||
UID: nameUID,
|
||||
}
|
||||
|
||||
return workflowTemplate, nil
|
||||
}
|
||||
|
||||
// GetManifestBytes returns the manifest as []byte
|
||||
func (wt *WorkflowTemplate) GetManifestBytes() []byte {
|
||||
return []byte(wt.Manifest)
|
||||
|
||||
@@ -51,6 +51,22 @@ func (wt *WorkspaceTemplate) GenerateUID(name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateWorkspaceTemplate creates a new workspace template with the given name.
|
||||
// All fields that can be generated in memory without external requests are filled out, such as the UID.
|
||||
func CreateWorkspaceTemplate(name string) (*WorkspaceTemplate, error) {
|
||||
nameUID, err := uid2.GenerateUID(name, 30)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
workspaceTemplate := &WorkspaceTemplate{
|
||||
Name: name,
|
||||
UID: nameUID,
|
||||
}
|
||||
|
||||
return workspaceTemplate, nil
|
||||
}
|
||||
|
||||
// InjectRuntimeParameters will inject all runtime variables into the WorkflowTemplate's manifest.
|
||||
func (wt *WorkspaceTemplate) InjectRuntimeParameters(config SystemConfig) error {
|
||||
if wt.WorkflowTemplate == nil {
|
||||
|
||||
Reference in New Issue
Block a user