Compare commits

...

37 Commits

Author SHA1 Message Date
Rush Tehrani
0d4f5fbb37 Merge pull request #678 from rushtehrani/docs/readme
docs: Update README
2020-10-20 14:11:01 -07:00
rushtehrani
43a4d60b5b add onepanel.gif 2020-10-20 13:43:57 -07:00
rushtehrani
6c85a81709 docs: Update README 2020-10-20 13:40:34 -07:00
Aleksandr Melnikov
e9d8a6ebe0 Merge pull request #677 from Vafilor/fix/artifact.repository.generic
fix: non-generic filesyncer for latest cvat migration
2020-10-20 11:17:54 -07:00
Andrey Melnikov
c8c0b6ffd4 fix: non-generic filesyncer for latest cvat migration 2020-10-20 11:03:03 -07:00
Andrey Melnikov
38780c2e08 Merge pull request #675 from Vafilor/feat/cvat.upgrade
feat: upgrade cvat to v14
2020-10-19 10:52:51 -07:00
Andrey Melnikov
e1f8ee846c feat: update-cvat 2020-10-19 10:48:10 -07:00
Andrey Melnikov
2cc4cbd6f8 cleanup: simplified updating workspace template into a separate function 2020-10-16 17:33:35 -07:00
Andrey Melnikov
276aaf6e7a feat: placeholder for updating cvat template. 2020-10-16 17:20:27 -07:00
Rush Tehrani
6e1a08fdc3 Merge pull request #673 from Vafilor/feat/gcs.compatible.files
fix: issue where files were not being listed when using gcs
2020-10-16 16:47:18 -07:00
Andrey Melnikov
7129fdf55f fix: issue where s3 with gcs wasn't working because of incompatible listing of files. 2020-10-16 16:41:49 -07:00
Rush Tehrani
00e3247fcd Merge pull request #672 from rushtehrani/fix/sys-uid-env-var
fix: Use sys-uid instead of sys-name for ONEPANEL_RESOURCE_UID
2020-10-16 12:58:23 -07:00
rushtehrani
c26019b4d5 use sys-uid instead of sys-name for ONEPANEL_RESOURCE_UID 2020-10-16 12:55:12 -07:00
Andrey Melnikov
acadc0c0a7 Merge pull request #670 from Vafilor/feat/tag.channel
fix: change notify slack channel to be org instead of dev
2020-10-16 09:57:46 -07:00
Rush Tehrani
1b97099d11 Merge pull request #669 from Vafilor/fix/db.connections
fix: use system-wide db connection for auth
2020-10-16 09:55:41 -07:00
Andrey Melnikov
c5365975ac fix: change notify slack channel to be org instead of dev 2020-10-16 09:53:45 -07:00
Andrey Melnikov
b7d37586a8 fix: add method to get client with a provided db 2020-10-16 09:45:51 -07:00
Andrey Melnikov
2bd3c3dde0 fix: use system-wide connection when checking authentication 2020-10-16 09:45:29 -07:00
Rush Tehrani
957313423c Merge pull request #668 from Vafilor/fix/db.connections
fix: close db connection when not in use.
2020-10-16 09:21:32 -07:00
Andrey Melnikov
c883f69fc9 fix: close db connection when not in use. 2020-10-15 21:48:25 -07:00
Andrey Melnikov
6c5b6c877e Merge pull request #660 from Vafilor/feat/revert.jwt
feat: revert jwt changes as it creates issues with workflow authentication logic
2020-10-14 12:15:27 -07:00
Andrey Melnikov
67e684a715 revert: remove util/tokens/jwt as it is no longer used. 2020-10-14 11:54:23 -07:00
Andrey Melnikov
20c4950b69 feat: revert jwt token from auth 2020-10-14 11:53:05 -07:00
Andrey Melnikov
5e5c3cca67 Merge pull request #657 from rushtehrani/fix/workspace-resource
fix: Update label function to return group and resource
2020-10-13 16:15:28 -07:00
Andrey Melnikov
575a33c272 Merge pull request #658 from Vafilor/feat/onepanel.io.653-additional-fixes
fix: issue with concurrent map access
2020-10-13 16:09:22 -07:00
rushtehrani
8e9b95aa12 update label function to return group and resource 2020-10-13 16:06:08 -07:00
Andrey Melnikov
38f1aafaec fix: issue with concurrent map access 2020-10-13 16:02:40 -07:00
rushtehrani
30ebda4918 use workspaces instead of statefulset for resource 2020-10-13 15:47:20 -07:00
Andrey Melnikov
ce972f2988 Merge pull request #656 from Vafilor/feat/onepanel.io.653-additional-fixes
fix: bad token issue
2020-10-13 15:27:45 -07:00
Andrey Melnikov
ede4c67c8f fix: bad token issue 2020-10-13 15:17:56 -07:00
Rush Tehrani
68ddec78c8 Merge pull request #655 from Vafilor/feat/onepanelio.core.653-auth.changes
feat: update incoming token to be a JWT token that takes in username
2020-10-13 12:45:14 -07:00
Andrey Melnikov
c42997a643 fix: error to be API friendly 2020-10-13 12:21:32 -07:00
Andrey Melnikov
5bd2feaa86 update: added username to returned auth token 2020-10-12 17:01:26 -07:00
Andrey Melnikov
de4302d226 chore: formatting updates for error strings 2020-10-12 16:17:16 -07:00
Andrey Melnikov
7150f24631 feat: update incoming token to be a JWT token that takes in username 2020-10-12 16:07:12 -07:00
Rush Tehrani
0e1e48dfc8 Merge pull request #647 from rushtehrani/master
fix: Use correct group for workspaces resource
2020-10-08 20:03:51 -07:00
rushtehrani
dd0f1f7705 use correct group for workspaces resource 2020-10-08 19:52:48 -07:00
21 changed files with 510 additions and 206 deletions

View File

@@ -22,7 +22,7 @@ jobs:
- name: Notify Slack Channels - name: Notify Slack Channels
uses: rtCamp/action-slack-notify@v2.0.0 uses: rtCamp/action-slack-notify@v2.0.0
env: env:
SLACK_CHANNEL: dev SLACK_CHANNEL: org
SLACK_ICON: https://avatars1.githubusercontent.com/u/30390575?s=48&v=4 SLACK_ICON: https://avatars1.githubusercontent.com/u/30390575?s=48&v=4
SLACK_TITLE: New Core Version SLACK_TITLE: New Core Version
SLACK_USERNAME: opBot SLACK_USERNAME: opBot

View File

@@ -9,57 +9,34 @@
[![chat](https://img.shields.io/badge/support-slack-01579b)](https://onepanel-ce.slack.com/join/shared_invite/zt-eyjnwec0-nLaHhjif9Y~gA05KuX6AUg#/) [![chat](https://img.shields.io/badge/support-slack-01579b)](https://onepanel-ce.slack.com/join/shared_invite/zt-eyjnwec0-nLaHhjif9Y~gA05KuX6AUg#/)
[![license](https://img.shields.io/github/license/onepanelio/core?color=01579b)](https://opensource.org/licenses/Apache-2.0) [![license](https://img.shields.io/github/license/onepanelio/core?color=01579b)](https://opensource.org/licenses/Apache-2.0)
Production scale, Kubernetes-native vision AI platform, with fully integrated components for model building, automated labeling, data processing and model training pipelines. Production scale, vision AI platform, with fully integrated components for model building, automated labeling, data processing and model training pipelines.
<img width="100%" src="img/onepanel.gif">
## Why Onepanel? ## Why Onepanel?
- End-to-end workflow and infrastructure automation for production scale vision AI - End-to-end automation for production scale vision AI pipelines
- Automatic resource management and on-demand scaling of CPU and GPU nodes - Best of breed, open source deep learning tools seamlessly integrated in one unified platform
- Easily scale your data processing and training pipelines to multiple nodes - Infrastructure automation so you can easily scale your data processing and training pipelines to multiple nodes
- Collaborate on all your deep learning tools and workflows through a unified web interface and SDKs - Customizable, reproducible and version controlled tooling and pipeline templates
- Scalability, flexibility and resiliency of Kubernetes without the deployment and configuration complexities - Scalability, flexibility and resiliency of Kubernetes without the deployment and configuration complexities
## Features ## Features
<table> - Annotate images and video with automatic annotation of bounding boxes and polygon masks, integrated with training pipelines to iteratively improve models for pre-annotation and inference.
<tr> - JupyterLab configured with extensions for debugging, Git/GitHub, notebook diffing and TensorBoard and support for Conda, OpenCV, Tensorflow and PyTorch with GPU and <a href="https://github.com/onepanelio/templates/tree/master/workspaces/jupyterlab">much more</a>.
<td width="50%" align="center"> - Build fully reproducible, distributed and parallel data processing and training pipelines with real-time logs and output snapshots.
<h3>Image and video annotation with automatic annotation</h3> - Bring your own IDEs, annotation tools and pipelines with a version controlled YAML and Docker based template engine.
<img width="100%" src="img/auto-annotation.gif">
<p>
Annotate images and video with automatic annotation of bounding boxes and polygon masks, integrated with training pipelines to iteratively improve models for pre-annotation and inference
</p>
</td>
<td width="50%" align="center">
<h3>JupyterLab with TensorFlow, PyTorch and GPU support</h3>
<img width="100%" src="img/jupyterlab.gif">
<p>
JupyterLab configured with extensions for debugging, Git/GitHub, notebook diffing and TensorBoard and support for Conda, OpenCV, Tensorflow and PyTorch with GPU and <a href="https://github.com/onepanelio/templates/tree/master/workspaces/jupyterlab">much more</a>
</p>
</td>
</tr>
<tr>
<td width="50%" align="center">
<h3>Auto scaling, distributed and parallel data processing and training pipelines</h3>
<img width="100%" src="img/pipelines.gif">
<p>
Build fully reproducible, distributed and parallel data processing and training pipelines with real-time logs and output snapshots
</p>
</td>
<td width="50%" align="center">
<h3>Version controlled pipelines and environments as code</h3>
<img width="100%" src="img/tools.gif">
<p>
Bring your own IDEs, annotation tools and pipelines with a version controlled YAML and Docker based template engine
</p>
</td>
</tr>
</table>
- Track and visualize model metrics and experiments with TensorBoard or bring your own experiment tracking tools. - Track and visualize model metrics and experiments with TensorBoard or bring your own experiment tracking tools.
- Access and share tools like AirSim, Carla, Gazebo or OpenAI Gym through your browser with VNC enabled workspaces. - Extend Onepanel with powerful REST APIs and SDKs to further automate your workflows.
- Extend Onepanel with powerful REST APIs and SDKs to further automate your pipelines and environments.
- Workflows, environments and infrastructure are all defined as code and version controlled, making them reproducible and portable. ## Online demo
- Powered by Kubernetes so you can deploy anywhere Kubernetes can run. We have created an [online demo environment](https://onepanel.typeform.com/to/kQfDX5Vf?product=github) so that you can quickly try Onepanel.
Note that this is a shared demo environment with the following restrictions:
- Data is reset every few hours
- One type of node pool (machine type) with a limit of 5 concurrent nodes
- Certain actions may be restricted
## Quick start ## Quick start
See [quick start guide](https://docs.onepanel.ai/docs/getting-started/quickstart) to get started with the platform of your choice. See [quick start guide](https://docs.onepanel.ai/docs/getting-started/quickstart) to get started with the platform of your choice.
@@ -74,17 +51,17 @@ See [documentation](https://docs.onepanel.ai) to get started or for more detaile
To submit a feature request, report a bug or documentation issue, please open a GitHub [pull request](https://github.com/onepanelio/core/pulls) or [issue](https://github.com/onepanelio/core/issues). To submit a feature request, report a bug or documentation issue, please open a GitHub [pull request](https://github.com/onepanelio/core/pulls) or [issue](https://github.com/onepanelio/core/issues).
For help, questions, release announcements and contribution discussions, join us on [Slack](https://join.slack.com/t/onepanel-ce/shared_invite/zt-eyjnwec0-nLaHhjif9Y~gA05KuX6AUg). For help, questions, release announcements and contribution discussions, join us on [GitHub discussions](https://github.com/onepanelio/core/discussions).
## Contributing ## Contributing
Onepanel is modular and consists of the following repositories: Onepanel is modular and consists of the following repositories:
[Core API](https://github.com/onepanelio/core/) (this repository) - Code base for backend (Go)\ [Backend](https://github.com/onepanelio/core/) (this repository) - Code base for backend (Go)\
[Core UI](https://github.com/onepanelio/core-ui/) - Code base for UI (Angular + TypeScript)\ [Frontend](https://github.com/onepanelio/core-ui/) - Code base for frontend (Angular + TypeScript)\
[CLI](https://github.com/onepanelio/cli/) - Code base for Go CLI for installation and management (Go)\ [CLI](https://github.com/onepanelio/cli/) - Code base for installation and management CLI (Go)\
[Manifests](https://github.com/onepanelio/core-ui/) - Kustomize manifests used by CLI for installation and management (YAML)\ [Manifests](https://github.com/onepanelio/core-ui/) - Kustomize manifests used by installation and management CLI (YAML)\
[Python SDK](https://github.com/onepanelio/python-sdk/) - Python SDK code and documentation\ [Python SDK](https://github.com/onepanelio/python-sdk/) - Python SDK code and documentation (Python)\
[Templates](https://github.com/onepanelio/templates) - Various Workspace, Workflow, Task and Sidecar Templates\ [Templates](https://github.com/onepanelio/templates) - Various Workspace, Workflow, Task and Sidecar Templates\
[Documentation](https://github.com/onepanelio/core-docs/) - The repository for documentation site\ [Documentation](https://github.com/onepanelio/core-docs/) - The repository for documentation site\
[API Documentation](https://github.com/onepanelio/core-api-docs/) - API documentation if you choose to use the API directly [API Documentation](https://github.com/onepanelio/core-api-docs/) - API documentation if you choose to use the API directly

View File

@@ -3,7 +3,7 @@
"info": { "info": {
"title": "Onepanel", "title": "Onepanel",
"description": "Onepanel API", "description": "Onepanel API",
"version": "0.13.0", "version": "0.14.0",
"contact": { "contact": {
"name": "Onepanel project", "name": "Onepanel project",
"url": "https://github.com/onepanelio/core" "url": "https://github.com/onepanelio/core"
@@ -77,7 +77,7 @@
"in": "body", "in": "body",
"required": true, "required": true,
"schema": { "schema": {
"$ref": "#/definitions/TokenWrapper" "$ref": "#/definitions/IsValidTokenRequest"
} }
} }
], ],
@@ -2967,11 +2967,28 @@
} }
} }
}, },
"IsValidTokenRequest": {
"type": "object",
"properties": {
"username": {
"type": "string"
},
"token": {
"type": "string"
}
}
},
"IsValidTokenResponse": { "IsValidTokenResponse": {
"type": "object", "type": "object",
"properties": { "properties": {
"domain": { "domain": {
"type": "string" "type": "string"
},
"token": {
"type": "string"
},
"username": {
"type": "string"
} }
} }
}, },
@@ -3394,14 +3411,6 @@
} }
} }
}, },
"TokenWrapper": {
"type": "object",
"properties": {
"token": {
"type": "string"
}
}
},
"UpdateSecretKeyValueResponse": { "UpdateSecretKeyValueResponse": {
"type": "object", "type": "object",
"properties": { "properties": {

View File

@@ -204,65 +204,19 @@ func (x *IsAuthorizedResponse) GetAuthorized() bool {
return false return false
} }
type TokenWrapper struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Token string `protobuf:"bytes,1,opt,name=token,proto3" json:"token,omitempty"`
}
func (x *TokenWrapper) Reset() {
*x = TokenWrapper{}
if protoimpl.UnsafeEnabled {
mi := &file_auth_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *TokenWrapper) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*TokenWrapper) ProtoMessage() {}
func (x *TokenWrapper) ProtoReflect() protoreflect.Message {
mi := &file_auth_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use TokenWrapper.ProtoReflect.Descriptor instead.
func (*TokenWrapper) Descriptor() ([]byte, []int) {
return file_auth_proto_rawDescGZIP(), []int{3}
}
func (x *TokenWrapper) GetToken() string {
if x != nil {
return x.Token
}
return ""
}
type IsValidTokenRequest struct { type IsValidTokenRequest struct {
state protoimpl.MessageState state protoimpl.MessageState
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields
Token *TokenWrapper `protobuf:"bytes,1,opt,name=token,proto3" json:"token,omitempty"` Username string `protobuf:"bytes,1,opt,name=username,proto3" json:"username,omitempty"`
Token string `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"`
} }
func (x *IsValidTokenRequest) Reset() { func (x *IsValidTokenRequest) Reset() {
*x = IsValidTokenRequest{} *x = IsValidTokenRequest{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_auth_proto_msgTypes[4] mi := &file_auth_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
} }
@@ -275,7 +229,7 @@ func (x *IsValidTokenRequest) String() string {
func (*IsValidTokenRequest) ProtoMessage() {} func (*IsValidTokenRequest) ProtoMessage() {}
func (x *IsValidTokenRequest) ProtoReflect() protoreflect.Message { func (x *IsValidTokenRequest) ProtoReflect() protoreflect.Message {
mi := &file_auth_proto_msgTypes[4] mi := &file_auth_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
@@ -288,14 +242,21 @@ func (x *IsValidTokenRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use IsValidTokenRequest.ProtoReflect.Descriptor instead. // Deprecated: Use IsValidTokenRequest.ProtoReflect.Descriptor instead.
func (*IsValidTokenRequest) Descriptor() ([]byte, []int) { func (*IsValidTokenRequest) Descriptor() ([]byte, []int) {
return file_auth_proto_rawDescGZIP(), []int{4} return file_auth_proto_rawDescGZIP(), []int{3}
} }
func (x *IsValidTokenRequest) GetToken() *TokenWrapper { func (x *IsValidTokenRequest) GetUsername() string {
if x != nil {
return x.Username
}
return ""
}
func (x *IsValidTokenRequest) GetToken() string {
if x != nil { if x != nil {
return x.Token return x.Token
} }
return nil return ""
} }
type IsValidTokenResponse struct { type IsValidTokenResponse struct {
@@ -303,13 +264,15 @@ type IsValidTokenResponse struct {
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields
Domain string `protobuf:"bytes,1,opt,name=domain,proto3" json:"domain,omitempty"` Domain string `protobuf:"bytes,1,opt,name=domain,proto3" json:"domain,omitempty"`
Token string `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"`
Username string `protobuf:"bytes,3,opt,name=username,proto3" json:"username,omitempty"`
} }
func (x *IsValidTokenResponse) Reset() { func (x *IsValidTokenResponse) Reset() {
*x = IsValidTokenResponse{} *x = IsValidTokenResponse{}
if protoimpl.UnsafeEnabled { if protoimpl.UnsafeEnabled {
mi := &file_auth_proto_msgTypes[5] mi := &file_auth_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi) ms.StoreMessageInfo(mi)
} }
@@ -322,7 +285,7 @@ func (x *IsValidTokenResponse) String() string {
func (*IsValidTokenResponse) ProtoMessage() {} func (*IsValidTokenResponse) ProtoMessage() {}
func (x *IsValidTokenResponse) ProtoReflect() protoreflect.Message { func (x *IsValidTokenResponse) ProtoReflect() protoreflect.Message {
mi := &file_auth_proto_msgTypes[5] mi := &file_auth_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil { if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil { if ms.LoadMessageInfo() == nil {
@@ -335,7 +298,7 @@ func (x *IsValidTokenResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use IsValidTokenResponse.ProtoReflect.Descriptor instead. // Deprecated: Use IsValidTokenResponse.ProtoReflect.Descriptor instead.
func (*IsValidTokenResponse) Descriptor() ([]byte, []int) { func (*IsValidTokenResponse) Descriptor() ([]byte, []int) {
return file_auth_proto_rawDescGZIP(), []int{5} return file_auth_proto_rawDescGZIP(), []int{4}
} }
func (x *IsValidTokenResponse) GetDomain() string { func (x *IsValidTokenResponse) GetDomain() string {
@@ -345,6 +308,20 @@ func (x *IsValidTokenResponse) GetDomain() string {
return "" return ""
} }
func (x *IsValidTokenResponse) GetToken() string {
if x != nil {
return x.Token
}
return ""
}
func (x *IsValidTokenResponse) GetUsername() string {
if x != nil {
return x.Username
}
return ""
}
var File_auth_proto protoreflect.FileDescriptor var File_auth_proto protoreflect.FileDescriptor
var file_auth_proto_rawDesc = []byte{ var file_auth_proto_rawDesc = []byte{
@@ -370,32 +347,33 @@ var file_auth_proto_rawDesc = []byte{
0x7a, 0x65, 0x64, 0x22, 0x36, 0x0a, 0x14, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x22, 0x36, 0x0a, 0x14, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69,
0x7a, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x61,
0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52,
0x0a, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x22, 0x24, 0x0a, 0x0c, 0x54, 0x0a, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x22, 0x47, 0x0a, 0x13, 0x49,
0x6f, 0x6b, 0x65, 0x6e, 0x57, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65,
0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
0x6e, 0x22, 0x3e, 0x0a, 0x13, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14,
0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74,
0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x54, 0x6f, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x60, 0x0a, 0x14, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54,
0x6b, 0x65, 0x6e, 0x57, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06,
0x6e, 0x22, 0x2e, 0x0a, 0x14, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f,
0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20,
0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73,
0x6e, 0x32, 0xea, 0x01, 0x0a, 0x0b, 0x41, 0x75, 0x74, 0x68, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73,
0x65, 0x12, 0x6c, 0x0a, 0x0c, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x32, 0xe6, 0x01, 0x0a, 0x0b, 0x41, 0x75, 0x74, 0x68, 0x53,
0x6e, 0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x68, 0x0a, 0x0c, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69,
0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x56,
0x69, 0x2e, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x27, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x21, 0x22, 0x18, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f,
0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x61, 0x75, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x23, 0x82, 0xd3, 0xe4,
0x74, 0x68, 0x2f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x3a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x93, 0x02, 0x1d, 0x22, 0x18, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74,
0x6d, 0x0a, 0x0c, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x12, 0x61, 0x31, 0x2f, 0x61, 0x75, 0x74, 0x68, 0x2f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x3a, 0x01, 0x2a,
0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x12, 0x6d, 0x0a, 0x0c, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64,
0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69,
0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69,
0x6f, 0x6e, 0x73, 0x65, 0x22, 0x28, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x22, 0x22, 0x12, 0x2f, 0x61, 0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x73,
0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x61, 0x75, 0x74, 0x68, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x28, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x22, 0x22, 0x12, 0x2f,
0x3a, 0x0c, 0x69, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x62, 0x06, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x61, 0x75, 0x74,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x68, 0x3a, 0x0c, 0x69, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
} }
var ( var (
@@ -410,27 +388,25 @@ func file_auth_proto_rawDescGZIP() []byte {
return file_auth_proto_rawDescData return file_auth_proto_rawDescData
} }
var file_auth_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_auth_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
var file_auth_proto_goTypes = []interface{}{ var file_auth_proto_goTypes = []interface{}{
(*IsAuthorized)(nil), // 0: api.IsAuthorized (*IsAuthorized)(nil), // 0: api.IsAuthorized
(*IsAuthorizedRequest)(nil), // 1: api.IsAuthorizedRequest (*IsAuthorizedRequest)(nil), // 1: api.IsAuthorizedRequest
(*IsAuthorizedResponse)(nil), // 2: api.IsAuthorizedResponse (*IsAuthorizedResponse)(nil), // 2: api.IsAuthorizedResponse
(*TokenWrapper)(nil), // 3: api.TokenWrapper (*IsValidTokenRequest)(nil), // 3: api.IsValidTokenRequest
(*IsValidTokenRequest)(nil), // 4: api.IsValidTokenRequest (*IsValidTokenResponse)(nil), // 4: api.IsValidTokenResponse
(*IsValidTokenResponse)(nil), // 5: api.IsValidTokenResponse
} }
var file_auth_proto_depIdxs = []int32{ var file_auth_proto_depIdxs = []int32{
0, // 0: api.IsAuthorizedRequest.isAuthorized:type_name -> api.IsAuthorized 0, // 0: api.IsAuthorizedRequest.isAuthorized:type_name -> api.IsAuthorized
3, // 1: api.IsValidTokenRequest.token:type_name -> api.TokenWrapper 3, // 1: api.AuthService.IsValidToken:input_type -> api.IsValidTokenRequest
4, // 2: api.AuthService.IsValidToken:input_type -> api.IsValidTokenRequest 1, // 2: api.AuthService.IsAuthorized:input_type -> api.IsAuthorizedRequest
1, // 3: api.AuthService.IsAuthorized:input_type -> api.IsAuthorizedRequest 4, // 3: api.AuthService.IsValidToken:output_type -> api.IsValidTokenResponse
5, // 4: api.AuthService.IsValidToken:output_type -> api.IsValidTokenResponse 2, // 4: api.AuthService.IsAuthorized:output_type -> api.IsAuthorizedResponse
2, // 5: api.AuthService.IsAuthorized:output_type -> api.IsAuthorizedResponse 3, // [3:5] is the sub-list for method output_type
4, // [4:6] is the sub-list for method output_type 1, // [1:3] is the sub-list for method input_type
2, // [2:4] is the sub-list for method input_type 1, // [1:1] is the sub-list for extension type_name
2, // [2:2] is the sub-list for extension type_name 1, // [1:1] is the sub-list for extension extendee
2, // [2:2] is the sub-list for extension extendee 0, // [0:1] is the sub-list for field type_name
0, // [0:2] is the sub-list for field type_name
} }
func init() { file_auth_proto_init() } func init() { file_auth_proto_init() }
@@ -476,18 +452,6 @@ func file_auth_proto_init() {
} }
} }
file_auth_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { file_auth_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*TokenWrapper); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_auth_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*IsValidTokenRequest); i { switch v := v.(*IsValidTokenRequest); i {
case 0: case 0:
return &v.state return &v.state
@@ -499,7 +463,7 @@ func file_auth_proto_init() {
return nil return nil
} }
} }
file_auth_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { file_auth_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*IsValidTokenResponse); i { switch v := v.(*IsValidTokenResponse); i {
case 0: case 0:
return &v.state return &v.state
@@ -518,7 +482,7 @@ func file_auth_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(), GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_auth_proto_rawDesc, RawDescriptor: file_auth_proto_rawDesc,
NumEnums: 0, NumEnums: 0,
NumMessages: 6, NumMessages: 5,
NumExtensions: 0, NumExtensions: 0,
NumServices: 1, NumServices: 1,
}, },

View File

@@ -39,7 +39,7 @@ func request_AuthService_IsValidToken_0(ctx context.Context, marshaler runtime.M
if berr != nil { if berr != nil {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr) return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
} }
if err := marshaler.NewDecoder(newReader()).Decode(&protoReq.Token); err != nil && err != io.EOF { if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
} }
@@ -56,7 +56,7 @@ func local_request_AuthService_IsValidToken_0(ctx context.Context, marshaler run
if berr != nil { if berr != nil {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr) return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
} }
if err := marshaler.NewDecoder(newReader()).Decode(&protoReq.Token); err != nil && err != io.EOF { if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
} }

View File

@@ -9,7 +9,7 @@ service AuthService {
rpc IsValidToken(IsValidTokenRequest) returns (IsValidTokenResponse) { rpc IsValidToken(IsValidTokenRequest) returns (IsValidTokenResponse) {
option (google.api.http) = { option (google.api.http) = {
post: "/apis/v1beta1/auth/token" post: "/apis/v1beta1/auth/token"
body: "token" body: "*"
}; };
} }
@@ -37,14 +37,13 @@ message IsAuthorizedResponse {
bool authorized = 1; bool authorized = 1;
} }
message TokenWrapper {
string token = 1;
}
message IsValidTokenRequest { message IsValidTokenRequest {
TokenWrapper token = 1; string username = 1;
string token = 2;
} }
message IsValidTokenResponse { message IsValidTokenResponse {
string domain = 1; string domain = 1;
string token = 2;
string username = 3;
} }

View File

@@ -0,0 +1,147 @@
# Workspace arguments
arguments:
parameters:
- name: sync-directory
displayName: Directory to sync raw input and training output
value: workflow-data
hint: Location to sync raw input, models and checkpoints from default object storage. Note that this will be relative to the current namespace.
containers:
- name: cvat-db
image: postgres:10-alpine
env:
- name: POSTGRES_USER
value: root
- name: POSTGRES_DB
value: cvat
- name: POSTGRES_HOST_AUTH_METHOD
value: trust
- name: PGDATA
value: /var/lib/psql/data
ports:
- containerPort: 5432
name: tcp
volumeMounts:
- name: db
mountPath: /var/lib/psql
- name: cvat-redis
image: redis:4.0-alpine
ports:
- containerPort: 6379
name: tcp
- name: cvat
image: onepanel/cvat:0.14.0_cvat.1.0.0
env:
- name: DJANGO_MODWSGI_EXTRA_ARGS
value: ""
- name: ALLOWED_HOSTS
value: '*'
- name: CVAT_REDIS_HOST
value: localhost
- name: CVAT_POSTGRES_HOST
value: localhost
- name: CVAT_SHARE_URL
value: /home/django/data
- name: ONEPANEL_SYNC_DIRECTORY
value: '{{workspace.parameters.sync-directory}}'
- name: NVIDIA_VISIBLE_DEVICES
value: all
- name: NVIDIA_DRIVER_CAPABILITIES
value: compute,utility
- name: NVIDIA_REQUIRE_CUDA
value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
ports:
- containerPort: 8080
name: http
volumeMounts:
- name: data
mountPath: /home/django/data
- name: keys
mountPath: /home/django/keys
- name: logs
mountPath: /home/django/logs
- name: models
mountPath: /home/django/models
- name: share
mountPath: /home/django/share
- name: sys-namespace-config
mountPath: /etc/onepanel
readOnly: true
- name: cvat-ui
image: onepanel/cvat-ui:0.14.0_cvat.1.0.0
ports:
- containerPort: 80
name: http
# You can add multiple FileSyncer sidecar containers if needed
- name: filesyncer
image: onepanel/filesyncer:{{.ArtifactRepositoryType}}
imagePullPolicy: Always
args:
- download
- -server-prefix=/sys/filesyncer
env:
- name: FS_PATH
value: /mnt/share
- name: FS_PREFIX
value: '{{workflow.namespace}}/{{workspace.parameters.sync-directory}}'
volumeMounts:
- name: share
mountPath: /mnt/share
- name: sys-namespace-config
mountPath: /etc/onepanel
readOnly: true
ports:
- name: cvat-ui
port: 80
protocol: TCP
targetPort: 80
- name: cvat
port: 8080
protocol: TCP
targetPort: 8080
- name: fs
port: 8888
protocol: TCP
targetPort: 8888
routes:
- match:
- uri:
prefix: /sys/filesyncer
route:
- destination:
port:
number: 8888
- match:
- uri:
regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
- queryParams:
id:
regex: \d+.*
route:
- destination:
port:
number: 8080
- match:
- uri:
prefix: /
route:
- destination:
port:
number: 80
# DAG Workflow to be executed once a Workspace action completes (optional)
# Uncomment the lines below if you want to send Slack notifications
#postExecutionWorkflow:
# entrypoint: main
# templates:
# - name: main
# dag:
# tasks:
# - name: slack-notify
# template: slack-notify
# - name: slack-notify
# container:
# image: technosophos/slack-notify
# args:
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
# command:
# - sh
# - -c

View File

@@ -0,0 +1,25 @@
package migration
import (
"database/sql"
"github.com/pressly/goose"
)
func initialize20201016170415() {
if _, ok := initializedMigrations[20201016170415]; !ok {
goose.AddMigration(Up20201016170415, Down20201016170415)
initializedMigrations[20201016170415] = true
}
}
// Up20201016170415 updates cvat to a new version
func Up20201016170415(tx *sql.Tx) error {
// This code is executed when the migration is applied.
return updateWorkspaceTemplateManifest("cvat_20201016170415.yaml", cvatTemplateName)
}
// Down20201016170415 does nothing
func Down20201016170415(tx *sql.Tx) error {
// This code is executed when the migration is rolled back.
return nil
}

View File

@@ -5,7 +5,10 @@ import (
sq "github.com/Masterminds/squirrel" sq "github.com/Masterminds/squirrel"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
v1 "github.com/onepanelio/core/pkg" v1 "github.com/onepanelio/core/pkg"
"io/ioutil"
"log" "log"
"os"
"path/filepath"
"strings" "strings"
) )
@@ -60,6 +63,7 @@ func Initialize() {
initialize20200929144301() initialize20200929144301()
initialize20200929153931() initialize20200929153931()
initialize20201001070806() initialize20201001070806()
initialize20201016170415()
if err := client.DB.Close(); err != nil { if err := client.DB.Close(); err != nil {
log.Printf("[error] closing db %v", err) log.Printf("[error] closing db %v", err)
@@ -128,3 +132,17 @@ func ReplaceArtifactRepositoryType(client *v1.Client, namespace *v1.Namespace, w
return nil return nil
} }
// readDataFile returns the contents of a file in the db/data/{name} directory
func readDataFile(name string) (string, error) {
curDir, err := os.Getwd()
if err != nil {
return "", err
}
data, err := ioutil.ReadFile(filepath.Join(curDir, "db", "data", name))
if err != nil {
return "", err
}
return string(data), nil
}

49
db/go/util.go Normal file
View File

@@ -0,0 +1,49 @@
package migration
import (
v1 "github.com/onepanelio/core/pkg"
uid2 "github.com/onepanelio/core/pkg/util/uid"
)
// updateWorkspaceTemplateManifest will update the workspace template given by {{templateName}} with the contents
// given by {{filename}}
// It will do so for all namespaces.
func updateWorkspaceTemplateManifest(filename, templateName string) error {
client, err := getClient()
if err != nil {
return err
}
defer client.DB.Close()
namespaces, err := client.ListOnepanelEnabledNamespaces()
if err != nil {
return err
}
newManifest, err := readDataFile(filename)
if err != nil {
return err
}
uid, err := uid2.GenerateUID(templateName, 30)
if err != nil {
return err
}
for _, namespace := range namespaces {
workspaceTemplate := &v1.WorkspaceTemplate{
UID: uid,
Name: templateName,
Manifest: newManifest,
}
err = ReplaceArtifactRepositoryType(client, namespace, nil, workspaceTemplate)
if err != nil {
return err
}
if _, err := client.UpdateWorkspaceTemplateManifest(namespace.Name, uid, workspaceTemplate.Manifest); err != nil {
return err
}
}
return nil
}

1
go.mod
View File

@@ -43,4 +43,5 @@ require (
k8s.io/apimachinery v0.16.7-beta.0 k8s.io/apimachinery v0.16.7-beta.0
k8s.io/client-go v0.16.4 k8s.io/client-go v0.16.4
sigs.k8s.io/yaml v1.2.0 sigs.k8s.io/yaml v1.2.0
github.com/dgrijalva/jwt-go v3.2.0+incompatible
) )

BIN
img/onepanel.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.7 MiB

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"flag" "flag"
"fmt" "fmt"
migrations "github.com/onepanelio/core/db/go"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
@@ -25,7 +26,6 @@ import (
"github.com/grpc-ecosystem/grpc-gateway/runtime" "github.com/grpc-ecosystem/grpc-gateway/runtime"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"github.com/onepanelio/core/api" "github.com/onepanelio/core/api"
migrations "github.com/onepanelio/core/db/go"
v1 "github.com/onepanelio/core/pkg" v1 "github.com/onepanelio/core/pkg"
"github.com/onepanelio/core/pkg/util/env" "github.com/onepanelio/core/pkg/util/env"
"github.com/onepanelio/core/server" "github.com/onepanelio/core/server"

View File

@@ -4,6 +4,7 @@ import (
"fmt" "fmt"
sq "github.com/Masterminds/squirrel" sq "github.com/Masterminds/squirrel"
argoprojv1alpha1 "github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1" argoprojv1alpha1 "github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
"github.com/jmoiron/sqlx"
"github.com/onepanelio/core/pkg/util/gcs" "github.com/onepanelio/core/pkg/util/gcs"
"github.com/onepanelio/core/pkg/util/router" "github.com/onepanelio/core/pkg/util/router"
"github.com/onepanelio/core/pkg/util/s3" "github.com/onepanelio/core/pkg/util/s3"
@@ -38,6 +39,37 @@ func NewConfig() (config *Config) {
return return
} }
// GetDefaultClient loads a default k8s client
func GetDefaultClient() (*Client, error) {
kubeConfig := NewConfig()
client, err := NewClient(kubeConfig, nil, nil)
if err != nil {
return nil, err
}
config, err := client.GetSystemConfig()
if err != nil {
return nil, err
}
dbDriverName, dbDataSourceName := config.DatabaseConnection()
client.DB = NewDB(sqlx.MustConnect(dbDriverName, dbDataSourceName))
return client, nil
}
// GetDefaultClientWithDB loads a default k8s client with an existing DB
func GetDefaultClientWithDB(db *DB) (*Client, error) {
kubeConfig := NewConfig()
client, err := NewClient(kubeConfig, nil, nil)
if err != nil {
return nil, err
}
client.DB = db
return client, nil
}
// NewClient creates a client to interact with the Onepanel system. // NewClient creates a client to interact with the Onepanel system.
// It includes access to the database, kubernetes, argo, and configuration. // It includes access to the database, kubernetes, argo, and configuration.
func NewClient(config *Config, db *DB, systemConfig SystemConfig) (client *Client, err error) { func NewClient(config *Config, db *DB, systemConfig SystemConfig) (client *Client, err error) {

View File

@@ -38,6 +38,12 @@ func NewSystemConfig(configMap *ConfigMap, secret *Secret) (config SystemConfig,
} }
config["databasePassword"] = string(databasePassword) config["databasePassword"] = string(databasePassword)
hmac, err := base64.StdEncoding.DecodeString(secret.Data["hmac"])
if err != nil {
return
}
config["hmac"] = string(hmac)
return return
} }
@@ -183,6 +189,16 @@ func (s SystemConfig) UpdateNodePoolOptions(parameters []Parameter) ([]Parameter
return result, nil return result, nil
} }
// HMACKey gets the HMAC value, or nil.
func (s SystemConfig) HMACKey() []byte {
hmac := s.GetValue("hmac")
if hmac == nil {
return []byte{}
}
return []byte(*hmac)
}
// ArtifactRepositoryS3Provider is meant to be used // ArtifactRepositoryS3Provider is meant to be used
// by the CLI. CLI will marshal this struct into the correct // by the CLI. CLI will marshal this struct into the correct
// YAML structure for k8s configmap / secret. // YAML structure for k8s configmap / secret.

View File

@@ -1340,7 +1340,7 @@ func (c *Client) ListFiles(namespace, key string) (files []*File, err error) {
doneCh := make(chan struct{}) doneCh := make(chan struct{})
defer close(doneCh) defer close(doneCh)
for objInfo := range s3Client.ListObjectsV2(config.ArtifactRepository.S3.Bucket, key, false, doneCh) { for objInfo := range s3Client.ListObjects(config.ArtifactRepository.S3.Bucket, key, false, doneCh) {
if objInfo.Key == key { if objInfo.Key == key {
continue continue
} }
@@ -1851,14 +1851,14 @@ func workflowExecutionsSelectBuilderNoColumns(namespace, workflowTemplateUID, wo
func workflowExecutionsSelectBuilder(namespace, workflowTemplateUID, workflowTemplateVersion string, includeSystem bool) sq.SelectBuilder { func workflowExecutionsSelectBuilder(namespace, workflowTemplateUID, workflowTemplateVersion string, includeSystem bool) sq.SelectBuilder {
sb := workflowExecutionsSelectBuilderNoColumns(namespace, workflowTemplateUID, workflowTemplateVersion, includeSystem) sb := workflowExecutionsSelectBuilderNoColumns(namespace, workflowTemplateUID, workflowTemplateVersion, includeSystem)
sb = sb.Columns(getWorkflowExecutionColumns("we", "")...). sb = sb.Columns(getWorkflowExecutionColumns("we")...).
Columns(`wtv.version "workflow_template.version"`, `wtv.created_at "workflow_template.created_at"`, `wt.name "workflow_template.name"`, `wt.uid "workflow_template.uid"`) Columns(`wtv.version "workflow_template.version"`, `wtv.created_at "workflow_template.created_at"`, `wt.name "workflow_template.name"`, `wt.uid "workflow_template.uid"`)
return sb return sb
} }
func (c *Client) getWorkflowExecutionAndTemplate(namespace string, uid string) (workflow *WorkflowExecution, err error) { func (c *Client) getWorkflowExecutionAndTemplate(namespace string, uid string) (workflow *WorkflowExecution, err error) {
sb := sb.Select(getWorkflowExecutionColumns("we", "")...). sb := sb.Select(getWorkflowExecutionColumns("we")...).
Columns(getWorkflowTemplateColumns("wt", "workflow_template")...). Columns(getWorkflowTemplateColumns("wt", "workflow_template")...).
Columns(`wtv.manifest "workflow_template.manifest"`, `wtv.version "workflow_template.version"`). Columns(`wtv.manifest "workflow_template.manifest"`, `wtv.version "workflow_template.version"`).
From("workflow_executions we"). From("workflow_executions we").

View File

@@ -401,7 +401,7 @@ func createStatefulSetManifest(spec *WorkspaceSpec, config map[string]string, se
env.PrependEnvVarToContainer(container, "ONEPANEL_DOMAIN", config["ONEPANEL_DOMAIN"]) env.PrependEnvVarToContainer(container, "ONEPANEL_DOMAIN", config["ONEPANEL_DOMAIN"])
env.PrependEnvVarToContainer(container, "ONEPANEL_PROVIDER", config["ONEPANEL_PROVIDER"]) env.PrependEnvVarToContainer(container, "ONEPANEL_PROVIDER", config["ONEPANEL_PROVIDER"])
env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_NAMESPACE", "{{workflow.namespace}}") env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_NAMESPACE", "{{workflow.namespace}}")
env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_UID", "{{workflow.parameters.sys-name}}") env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_UID", "{{workflow.parameters.sys-uid}}")
for _, service := range services { for _, service := range services {
envName := fmt.Sprintf("ONEPANEL_SERVICES_%v_API_URL", strings.ToUpper(service.Name)) envName := fmt.Sprintf("ONEPANEL_SERVICES_%v_API_URL", strings.ToUpper(service.Name))

View File

@@ -2,9 +2,13 @@ package auth
import ( import (
"context" "context"
"crypto/md5"
"encoding/hex"
"errors" "errors"
"fmt" "fmt"
"github.com/onepanelio/core/api" "github.com/onepanelio/core/api"
"github.com/onepanelio/core/pkg/util"
v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
"net/http" "net/http"
"strings" "strings"
@@ -36,6 +40,9 @@ func getBearerToken(ctx context.Context) (*string, bool) {
return nil, false return nil, false
} }
t = strings.ReplaceAll(t, prefix, "") t = strings.ReplaceAll(t, prefix, "")
if t == "null" {
return nil, false
}
return &t, true return &t, true
} }
@@ -64,7 +71,12 @@ func getClient(ctx context.Context, kubeConfig *v1.Config, db *v1.DB, sysConfig
return nil, status.Error(codes.Unauthenticated, `Missing or invalid "authorization" header.`) return nil, status.Error(codes.Unauthenticated, `Missing or invalid "authorization" header.`)
} }
if sysConfig["token"] != *bearerToken {
sysConfig["token"] = *bearerToken
}
kubeConfig.BearerToken = *bearerToken kubeConfig.BearerToken = *bearerToken
client, err := v1.NewClient(kubeConfig, db, sysConfig) client, err := v1.NewClient(kubeConfig, db, sysConfig)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -98,6 +110,43 @@ func IsAuthorized(c *v1.Client, namespace, verb, group, resource, name string) (
return return
} }
func verifyLogin(client *v1.Client, tokenRequest *api.IsValidTokenRequest) (rawToken string, err error) {
accountsList, err := client.CoreV1().ServiceAccounts("onepanel").List(v1.ListOptions{})
if err != nil {
return "", err
}
authTokenSecretName := ""
for _, serviceAccount := range accountsList.Items {
if serviceAccount.Name != tokenRequest.Username {
continue
}
for _, secret := range serviceAccount.Secrets {
if strings.Contains(secret.Name, "-token-") {
authTokenSecretName = secret.Name
break
}
}
}
if authTokenSecretName == "" {
return "", util.NewUserError(codes.InvalidArgument, fmt.Sprintf("unknown service account '%v'", tokenRequest.Username))
}
secret, err := client.CoreV1().Secrets("onepanel").Get(authTokenSecretName, v12.GetOptions{})
if err != nil {
return "", err
}
currentTokenBytes := md5.Sum(secret.Data["token"])
currentTokenString := hex.EncodeToString(currentTokenBytes[:])
if tokenRequest.Token != fmt.Sprintf("%s", currentTokenString) {
return "", util.NewUserError(codes.InvalidArgument, "token doesn't match what's on record")
}
return string(secret.Data["token"]), nil
}
// UnaryInterceptor performs authentication checks. // UnaryInterceptor performs authentication checks.
// The two main cases are: // The two main cases are:
// 1. Is the token valid? This is used for logging in. // 1. Is the token valid? This is used for logging in.
@@ -113,10 +162,25 @@ func UnaryInterceptor(kubeConfig *v1.Config, db *v1.DB, sysConfig v1.SystemConfi
tokenRequest, ok := req.(*api.IsValidTokenRequest) tokenRequest, ok := req.(*api.IsValidTokenRequest)
if !ok { if !ok {
return resp, errors.New("IsValidToken does not have correct request type") return resp, errors.New("LogInRequest does not have correct request type")
} }
md.Set("authorization", tokenRequest.Token.Token) defaultClient, err := v1.GetDefaultClientWithDB(db)
if err != nil {
return nil, err
}
rawToken, err := verifyLogin(defaultClient, tokenRequest)
if err != nil {
return nil, err
}
sysConfig, err := defaultClient.GetSystemConfig()
if err != nil {
return nil, err
}
md.Set("onepanel-auth-token", rawToken)
ctx, err = getClient(ctx, kubeConfig, db, sysConfig) ctx, err = getClient(ctx, kubeConfig, db, sysConfig)
if err != nil { if err != nil {

View File

@@ -66,8 +66,10 @@ func (a *AuthServer) IsValidToken(ctx context.Context, req *api.IsValidTokenRequ
if err != nil { if err != nil {
return return
} }
res = &api.IsValidTokenResponse{} res = &api.IsValidTokenResponse{
res.Domain = config["ONEPANEL_DOMAIN"] Domain: config["ONEPANEL_DOMAIN"],
Token: config["token"],
}
return res, nil return res, nil
} }

View File

@@ -7,21 +7,22 @@ import (
"github.com/onepanelio/core/server/auth" "github.com/onepanelio/core/server/auth"
) )
func resourceIdentifierToArgoResource(identifier string) string { func getGroupAndResourceByIdentifier(identifier string) (group, resource string) {
group = "argoproj.io"
switch identifier { switch identifier {
case v1.TypeWorkflowTemplate: case v1.TypeWorkflowTemplate:
return "workflowtemplates" return group, "workflowtemplates"
case v1.TypeWorkflowTemplateVersion: case v1.TypeWorkflowTemplateVersion:
return "workflowtemplates" return group, "workflowtemplates"
case v1.TypeWorkflowExecution: case v1.TypeWorkflowExecution:
return "workflows" return group, "workflows"
case v1.TypeCronWorkflow: case v1.TypeCronWorkflow:
return "cronworkflows" return group, "cronworkflows"
case v1.TypeWorkspace: case v1.TypeWorkspace:
return "statefulset" return "onepanel.io", "workspaces"
} }
return "" return "", ""
} }
func mapLabelsToKeyValue(labels []*v1.Label) []*api.KeyValue { func mapLabelsToKeyValue(labels []*v1.Label) []*api.KeyValue {
@@ -54,10 +55,10 @@ func NewLabelServer() *LabelServer {
} }
func (s *LabelServer) GetLabels(ctx context.Context, req *api.GetLabelsRequest) (*api.GetLabelsResponse, error) { func (s *LabelServer) GetLabels(ctx context.Context, req *api.GetLabelsRequest) (*api.GetLabelsResponse, error) {
argoResource := resourceIdentifierToArgoResource(req.Resource) group, resource := getGroupAndResourceByIdentifier(req.Resource)
client := getClient(ctx) client := getClient(ctx)
allowed, err := auth.IsAuthorized(client, req.Namespace, "get", "argoproj.io", argoResource, "") allowed, err := auth.IsAuthorized(client, req.Namespace, "get", group, resource, "")
if err != nil || !allowed { if err != nil || !allowed {
return nil, err return nil, err
} }
@@ -73,10 +74,10 @@ func (s *LabelServer) GetLabels(ctx context.Context, req *api.GetLabelsRequest)
} }
func (s *LabelServer) AddLabels(ctx context.Context, req *api.AddLabelsRequest) (*api.GetLabelsResponse, error) { func (s *LabelServer) AddLabels(ctx context.Context, req *api.AddLabelsRequest) (*api.GetLabelsResponse, error) {
argoResource := resourceIdentifierToArgoResource(req.Resource) group, resource := getGroupAndResourceByIdentifier(req.Resource)
client := getClient(ctx) client := getClient(ctx)
allowed, err := auth.IsAuthorized(client, req.Namespace, "create", "argoproj.io", argoResource, "") allowed, err := auth.IsAuthorized(client, req.Namespace, "create", group, resource, "")
if err != nil || !allowed { if err != nil || !allowed {
return nil, err return nil, err
} }
@@ -97,10 +98,10 @@ func (s *LabelServer) AddLabels(ctx context.Context, req *api.AddLabelsRequest)
} }
func (s *LabelServer) ReplaceLabels(ctx context.Context, req *api.ReplaceLabelsRequest) (*api.GetLabelsResponse, error) { func (s *LabelServer) ReplaceLabels(ctx context.Context, req *api.ReplaceLabelsRequest) (*api.GetLabelsResponse, error) {
argoResource := resourceIdentifierToArgoResource(req.Resource) group, resource := getGroupAndResourceByIdentifier(req.Resource)
client := getClient(ctx) client := getClient(ctx)
allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "argoproj.io", argoResource, "") allowed, err := auth.IsAuthorized(client, req.Namespace, "update", group, resource, "")
if err != nil || !allowed { if err != nil || !allowed {
return nil, err return nil, err
} }
@@ -121,11 +122,11 @@ func (s *LabelServer) ReplaceLabels(ctx context.Context, req *api.ReplaceLabelsR
} }
func (s *LabelServer) DeleteLabel(ctx context.Context, req *api.DeleteLabelRequest) (*api.GetLabelsResponse, error) { func (s *LabelServer) DeleteLabel(ctx context.Context, req *api.DeleteLabelRequest) (*api.GetLabelsResponse, error) {
argoResource := resourceIdentifierToArgoResource(req.Resource) group, resource := getGroupAndResourceByIdentifier(req.Resource)
client := getClient(ctx) client := getClient(ctx)
// update verb here since we are not deleting the resource, but labels // update verb here since we are not deleting the resource, but labels
allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "argoproj.io", argoResource, "") allowed, err := auth.IsAuthorized(client, req.Namespace, "update", group, resource, "")
if err != nil || !allowed { if err != nil || !allowed {
return nil, err return nil, err
} }

View File

@@ -363,7 +363,7 @@ func (s *WorkspaceServer) RetryLastWorkspaceAction(ctx context.Context, req *api
func (s *WorkspaceServer) GetWorkspaceStatisticsForNamespace(ctx context.Context, req *api.GetWorkspaceStatisticsForNamespaceRequest) (*api.GetWorkspaceStatisticsForNamespaceResponse, error) { func (s *WorkspaceServer) GetWorkspaceStatisticsForNamespace(ctx context.Context, req *api.GetWorkspaceStatisticsForNamespaceRequest) (*api.GetWorkspaceStatisticsForNamespaceResponse, error) {
client := getClient(ctx) client := getClient(ctx)
allowed, err := auth.IsAuthorized(client, req.Namespace, "list", "argoproj.io", "workspaces", "") allowed, err := auth.IsAuthorized(client, req.Namespace, "list", "onepanel.io", "workspaces", "")
if err != nil || !allowed { if err != nil || !allowed {
return nil, err return nil, err
} }