zishuo/onepanel
1
0
Fork 0
mirror of https://github.com/onepanelio/onepanel.git synced 2025-09-26 17:51:13 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #933 from Vafilor/feat/filesyncer.update.migrations

Browse Source 
feat: added migrations to update filesyncer to version 1.0.0
This commit is contained in:
Andrey Melnikov
2021-07-20 13:13:58 -07:00
committed by GitHub
parent 5b6979302e 6f8e3f56e7
commit abd8d3cde0
6 changed files with 451 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
db/go/20210719190719_update_filesyncer.go Normal file
View File

@@ -0,0 +1,66 @@
package migration
import (
"database/sql"
"github.com/pressly/goose"
"path/filepath"
)
func initialize20210719190719() {
if _, ok := initializedMigrations[20210719190719]; !ok {
goose.AddMigration(Up20210719190719, Down20210719190719)
initializedMigrations[20210719190719] = true
}
}
// Up20210719190719 updates the workspace templates to use new v1.0.0 of filesyncer
func Up20210719190719(tx *sql.Tx) error {
// This code is executed when the migration is applied.
if err := updateWorkspaceTemplateManifest(
filepath.Join("workspaces", "cvat", "20210719190719.yaml"),
cvatTemplateName); err != nil {
return err
}
if err := updateWorkspaceTemplateManifest(
filepath.Join("workspaces", "jupyterlab", "20210719190719.yaml"),
jupyterLabTemplateName); err != nil {
return err
}
if err := updateWorkspaceTemplateManifest(
filepath.Join("workspaces", "vnc", "20210719190719.yaml"),
deepLearningDesktopTemplateName); err != nil {
return err
}
return updateWorkspaceTemplateManifest(
filepath.Join("workspaces", "vscode", "20210719190719.yaml"),
vscodeWorkspaceTemplateName)
}
// Down20210719190719 rolls back the change to update filesyncer
func Down20210719190719(tx *sql.Tx) error {
// This code is executed when the migration is rolled back.
if err := updateWorkspaceTemplateManifest(
filepath.Join("workspaces", "cvat", "20210323175655.yaml"),
cvatTemplateName); err != nil {
return err
}
if err := updateWorkspaceTemplateManifest(
filepath.Join("workspaces", "jupyterlab", "20210323175655.yaml"),
jupyterLabTemplateName); err != nil {
return err
}
if err := updateWorkspaceTemplateManifest(
filepath.Join("workspaces", "vnc", "20210414165510.yaml"),
deepLearningDesktopTemplateName); err != nil {
return err
}
return updateWorkspaceTemplateManifest(
filepath.Join("workspaces", "vscode", "20210323175655.yaml"),
vscodeWorkspaceTemplateName)
}

1
db/go/db.go
View File

@@ -95,6 +95,7 @@ func Initialize() {
initialize20210329171739()
initialize20210329194731()
initialize20210414165510()
initialize20210719190719()
if err := client.DB.Close(); err != nil {
log.Printf("[error] closing db %v", err)

134
db/yaml/workspaces/cvat/20210719190719.yaml Normal file
View File

@@ -0,0 +1,134 @@
containers:
- name: cvat-db
image: postgres:10-alpine
env:
- name: POSTGRES_USER
value: root
- name: POSTGRES_DB
value: cvat
- name: POSTGRES_HOST_AUTH_METHOD
value: trust
- name: PGDATA
value: /var/lib/psql/data
ports:
- containerPort: 5432
name: tcp
volumeMounts:
- name: db
mountPath: /var/lib/psql
- name: cvat-redis
image: redis:4.0-alpine
ports:
- containerPort: 6379
name: tcp
- name: cvat
image: onepanel/cvat:v0.19.0_cvat.1.0.0
env:
- name: DJANGO_MODWSGI_EXTRA_ARGS
value: ""
- name: ALLOWED_HOSTS
value: '*'
- name: CVAT_REDIS_HOST
value: localhost
- name: CVAT_POSTGRES_HOST
value: localhost
- name: CVAT_SHARE_URL
value: /cvat/data
- name: CVAT_SHARE_DIR
value: /share
- name: CVAT_DATA_DIR
value: /cvat/data
- name: CVAT_MEDIA_DATA_DIR
value: /cvat/data/data
- name: CVAT_KEYS_DIR
value: /cvat/data/keys
- name: CVAT_MODELS_DIR
value: /cvat/data/models
- name: CVAT_LOGS_DIR
value: /cvat/logs
- name: CVAT_ANNOTATIONS_OBJECT_STORAGE_PREFIX
value: 'artifacts/$(ONEPANEL_RESOURCE_NAMESPACE)/annotations/'
- name: CVAT_ONEPANEL_WORKFLOWS_LABEL
value: 'key=used-by,value=cvat'
- name: NVIDIA_VISIBLE_DEVICES
value: all
- name: NVIDIA_DRIVER_CAPABILITIES
value: compute,utility
- name: NVIDIA_REQUIRE_CUDA
value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
- name: ONEPANEL_MAIN_CONTAINER
value: 'true'
ports:
- containerPort: 8080
name: http
volumeMounts:
- name: cvat-data
mountPath: /cvat
- name: share
mountPath: /share
- name: sys-namespace-config
mountPath: /etc/onepanel
readOnly: true
- name: cvat-ui
image: onepanel/cvat-ui:v0.19.0_cvat.1.0.0
ports:
- containerPort: 80
name: http
- name: sys-filesyncer
image: onepanel/filesyncer:v1.0.0
imagePullPolicy: Always
args:
- server
- -server-prefix=/sys/filesyncer
volumeMounts:
- name: share
mountPath: /share
- name: sys-namespace-config
mountPath: /etc/onepanel
readOnly: true
ports:
- name: cvat-ui
port: 80
protocol: TCP
targetPort: 80
- name: cvat
port: 8080
protocol: TCP
targetPort: 8080
- name: fs
port: 8888
protocol: TCP
targetPort: 8888
routes:
- match:
- uri:
prefix: /sys/filesyncer
route:
- destination:
port:
number: 8888
- match:
- uri:
regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
- queryParams:
id:
regex: \d+.*
route:
- destination:
port:
number: 8080
- match:
- uri:
prefix: /
route:
- destination:
port:
number: 80
volumeClaimTemplates:
- metadata:
name: db
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 20Gi

101
db/yaml/workspaces/jupyterlab/20210719190719.yaml Normal file
View File

@@ -0,0 +1,101 @@
containers:
- name: jupyterlab
image: onepanel/dl:v0.20.0
command: ["/bin/bash", "-c", "pip install onepanel-sdk && start.sh LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64 jupyter lab --LabApp.token='' --LabApp.allow_remote_access=True --LabApp.allow_origin=\"*\" --LabApp.disable_check_xsrf=True --LabApp.trust_xheaders=True --LabApp.base_url=/ --LabApp.tornado_settings='{\"headers\":{\"Content-Security-Policy\":\"frame-ancestors * 'self'\"}}' --notebook-dir='/data' --allow-root"]
workingDir: /data
env:
- name: tornado
value: "'{'headers':{'Content-Security-Policy':\"frame-ancestors\ *\ 'self'\"}}'"
- name: TENSORBOARD_PROXY_URL
value: '//$(ONEPANEL_RESOURCE_UID)--$(ONEPANEL_RESOURCE_NAMESPACE).$(ONEPANEL_DOMAIN)/tensorboard'
- name: ONEPANEL_MAIN_CONTAINER
value: 'true'
ports:
- containerPort: 8888
name: jupyterlab
- containerPort: 6006
name: tensorboard
- containerPort: 8080
name: nni
volumeMounts:
- name: data
mountPath: /data
lifecycle:
postStart:
exec:
command:
- /bin/sh
- -c
- >
condayml="/data/.environment.yml";
jupytertxt="/data/.jupexported.txt";
if [ -f "$condayml" ]; then conda env update -f $condayml; fi;
if [ -f "$jupytertxt" ]; then cat $jupytertxt | xargs -n 1 jupyter labextension install --no-build && jupyter lab build --minimize=False; fi;
preStop:
exec:
command:
- /bin/sh
- -c
- >
conda env export > /data/.environment.yml -n base;
jupyter labextension list 1>/dev/null 2> /data/.jup.txt;
cat /data/.jup.txt | sed -n '2,$p' | awk 'sub(/v/,"@", $2){print $1$2}' > /data/.jupexported.txt;
- name: sys-filesyncer
image: onepanel/filesyncer:v1.0.0
imagePullPolicy: Always
args:
- server
- -host=localhost:8889
- -server-prefix=/sys/filesyncer
volumeMounts:
- name: data
mountPath: /data
- name: sys-namespace-config
mountPath: /etc/onepanel
readOnly: true
ports:
- name: jupyterlab
port: 80
protocol: TCP
targetPort: 8888
- name: tensorboard
port: 6006
protocol: TCP
targetPort: 6006
- name: nni
port: 8080
protocol: TCP
targetPort: 8080
- name: fs
port: 8889
protocol: TCP
targetPort: 8889
routes:
- match:
- uri:
prefix: /sys/filesyncer
route:
- destination:
port:
number: 8889
- match:
- uri:
prefix: /tensorboard
route:
- destination:
port:
number: 6006
- match:
- uri:
prefix: /nni
route:
- destination:
port:
number: 8080
- match:
- uri:
prefix: /
route:
- destination:
port:
number: 80

81
db/yaml/workspaces/vnc/20210719190719.yaml Normal file
View File

@@ -0,0 +1,81 @@
arguments:
parameters:
# parameter screen-resolution allows users to select screen resolution
- name: screen-resolution
value: 1680x1050
type: select.select
displayName: Screen Resolution
options:
- name: 1280x1024
value: 1280x1024
- name: 1680x1050
value: 1680x1050
- name: 2880x1800
value: 2880x1800
containers:
- name: ubuntu
image: onepanel/vnc:dl-vnc
env:
- name: VNC_PASSWORDLESS
value: true
- name: VNC_RESOLUTION
value: '{{workflow.parameters.screen-resolution}}'
ports:
- containerPort: 6901
name: vnc
volumeMounts:
- name: data
mountPath: /data
- name: sys-filesyncer
image: onepanel/filesyncer:v1.0.0
imagePullPolicy: Always
args:
- server
- -host=localhost:8889
- -server-prefix=/sys/filesyncer
volumeMounts:
- name: data
mountPath: /data
- name: sys-namespace-config
mountPath: /etc/onepanel
readOnly: true
ports:
- name: vnc
port: 80
protocol: TCP
targetPort: 6901
- name: fs
port: 8889
protocol: TCP
targetPort: 8889
routes:
- match:
- uri:
prefix: /sys/filesyncer
route:
- destination:
port:
number: 8889
- match:
- uri:
prefix: /
route:
- destination:
port:
number: 80
# DAG Workflow to be executed once a Workspace action completes (optional)
#postExecutionWorkflow:
# entrypoint: main
# templates:
# - name: main
# dag:
# tasks:
# - name: slack-notify
# template: slack-notify
# - name: slack-notify
# container:
# image: technosophos/slack-notify
# args:
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
# command:
# - sh

68
db/yaml/workspaces/vscode/20210719190719.yaml Normal file
View File

@@ -0,0 +1,68 @@
containers:
- name: vscode
image: onepanel/vscode:v0.20.0_code-server.3.9.1
command: ["/bin/bash", "-c", "pip install onepanel-sdk && /usr/bin/entrypoint.sh --bind-addr 0.0.0.0:8080 --auth none ."]
env:
- name: ONEPANEL_MAIN_CONTAINER
value: 'true'
ports:
- containerPort: 8080
name: vscode
volumeMounts:
- name: data
mountPath: /data
lifecycle:
postStart:
exec:
command:
- /bin/sh
- -c
- >
condayml="/data/.environment.yml";
vscodetxt="/data/.vscode-extensions.txt";
if [ -f "$condayml" ]; then conda env update -f $condayml; fi;
if [ -f "$vscodetxt" ]; then cat $vscodetxt | xargs -n 1 code-server --install-extension; fi;
preStop:
exec:
command:
- /bin/sh
- -c
- >
conda env export > /data/.environment.yml -n base;
code-server --list-extensions | tail -n +2 > /data/.vscode-extensions.txt;
- name: sys-filesyncer
image: onepanel/filesyncer:v1.0.0
imagePullPolicy: Always
args:
- server
- -server-prefix=/sys/filesyncer
volumeMounts:
- name: data
mountPath: /data
- name: sys-namespace-config
mountPath: /etc/onepanel
readOnly: true
ports:
- name: vscode
port: 8080
protocol: TCP
targetPort: 8080
- name: fs
port: 8888
protocol: TCP
targetPort: 8888
routes:
- match:
- uri:
prefix: /sys/filesyncer
route:
- destination:
port:
number: 8888
- match:
- uri:
prefix: /
route:
- destination:
port:
number: 8080