Compare commits

...

37 Commits

Author SHA1 Message Date
Rush Tehrani
0d4f5fbb37 Merge pull request #678 from rushtehrani/docs/readme
docs: Update README
2020-10-20 14:11:01 -07:00
rushtehrani
43a4d60b5b add onepanel.gif 2020-10-20 13:43:57 -07:00
rushtehrani
6c85a81709 docs: Update README 2020-10-20 13:40:34 -07:00
Aleksandr Melnikov
e9d8a6ebe0 Merge pull request #677 from Vafilor/fix/artifact.repository.generic
fix: non-generic filesyncer for latest cvat migration
2020-10-20 11:17:54 -07:00
Andrey Melnikov
c8c0b6ffd4 fix: non-generic filesyncer for latest cvat migration 2020-10-20 11:03:03 -07:00
Andrey Melnikov
38780c2e08 Merge pull request #675 from Vafilor/feat/cvat.upgrade
feat: upgrade cvat to v14
2020-10-19 10:52:51 -07:00
Andrey Melnikov
e1f8ee846c feat: update-cvat 2020-10-19 10:48:10 -07:00
Andrey Melnikov
2cc4cbd6f8 cleanup: simplified updating workspace template into a separate function 2020-10-16 17:33:35 -07:00
Andrey Melnikov
276aaf6e7a feat: placeholder for updating cvat template. 2020-10-16 17:20:27 -07:00
Rush Tehrani
6e1a08fdc3 Merge pull request #673 from Vafilor/feat/gcs.compatible.files
fix: issue where files were not being listed when using gcs
2020-10-16 16:47:18 -07:00
Andrey Melnikov
7129fdf55f fix: issue where s3 with gcs wasn't working because of incompatible listing of files. 2020-10-16 16:41:49 -07:00
Rush Tehrani
00e3247fcd Merge pull request #672 from rushtehrani/fix/sys-uid-env-var
fix: Use sys-uid instead of sys-name for ONEPANEL_RESOURCE_UID
2020-10-16 12:58:23 -07:00
rushtehrani
c26019b4d5 use sys-uid instead of sys-name for ONEPANEL_RESOURCE_UID 2020-10-16 12:55:12 -07:00
Andrey Melnikov
acadc0c0a7 Merge pull request #670 from Vafilor/feat/tag.channel
fix: change notify slack channel to be org instead of dev
2020-10-16 09:57:46 -07:00
Rush Tehrani
1b97099d11 Merge pull request #669 from Vafilor/fix/db.connections
fix: use system-wide db connection for auth
2020-10-16 09:55:41 -07:00
Andrey Melnikov
c5365975ac fix: change notify slack channel to be org instead of dev 2020-10-16 09:53:45 -07:00
Andrey Melnikov
b7d37586a8 fix: add method to get client with a provided db 2020-10-16 09:45:51 -07:00
Andrey Melnikov
2bd3c3dde0 fix: use system-wide connection when checking authentication 2020-10-16 09:45:29 -07:00
Rush Tehrani
957313423c Merge pull request #668 from Vafilor/fix/db.connections
fix: close db connection when not in use.
2020-10-16 09:21:32 -07:00
Andrey Melnikov
c883f69fc9 fix: close db connection when not in use. 2020-10-15 21:48:25 -07:00
Andrey Melnikov
6c5b6c877e Merge pull request #660 from Vafilor/feat/revert.jwt
feat: revert jwt changes as it creates issues with workflow authentication logic
2020-10-14 12:15:27 -07:00
Andrey Melnikov
67e684a715 revert: remove util/tokens/jwt as it is no longer used. 2020-10-14 11:54:23 -07:00
Andrey Melnikov
20c4950b69 feat: revert jwt token from auth 2020-10-14 11:53:05 -07:00
Andrey Melnikov
5e5c3cca67 Merge pull request #657 from rushtehrani/fix/workspace-resource
fix: Update label function to return group and resource
2020-10-13 16:15:28 -07:00
Andrey Melnikov
575a33c272 Merge pull request #658 from Vafilor/feat/onepanel.io.653-additional-fixes
fix: issue with concurrent map access
2020-10-13 16:09:22 -07:00
rushtehrani
8e9b95aa12 update label function to return group and resource 2020-10-13 16:06:08 -07:00
Andrey Melnikov
38f1aafaec fix: issue with concurrent map access 2020-10-13 16:02:40 -07:00
rushtehrani
30ebda4918 use workspaces instead of statefulset for resource 2020-10-13 15:47:20 -07:00
Andrey Melnikov
ce972f2988 Merge pull request #656 from Vafilor/feat/onepanel.io.653-additional-fixes
fix: bad token issue
2020-10-13 15:27:45 -07:00
Andrey Melnikov
ede4c67c8f fix: bad token issue 2020-10-13 15:17:56 -07:00
Rush Tehrani
68ddec78c8 Merge pull request #655 from Vafilor/feat/onepanelio.core.653-auth.changes
feat: update incoming token to be a JWT token that takes in username
2020-10-13 12:45:14 -07:00
Andrey Melnikov
c42997a643 fix: error to be API friendly 2020-10-13 12:21:32 -07:00
Andrey Melnikov
5bd2feaa86 update: added username to returned auth token 2020-10-12 17:01:26 -07:00
Andrey Melnikov
de4302d226 chore: formatting updates for error strings 2020-10-12 16:17:16 -07:00
Andrey Melnikov
7150f24631 feat: update incoming token to be a JWT token that takes in username 2020-10-12 16:07:12 -07:00
Rush Tehrani
0e1e48dfc8 Merge pull request #647 from rushtehrani/master
fix: Use correct group for workspaces resource
2020-10-08 20:03:51 -07:00
rushtehrani
dd0f1f7705 use correct group for workspaces resource 2020-10-08 19:52:48 -07:00
21 changed files with 510 additions and 206 deletions

View File

@@ -22,7 +22,7 @@ jobs:
- name: Notify Slack Channels
uses: rtCamp/action-slack-notify@v2.0.0
env:
SLACK_CHANNEL: dev
SLACK_CHANNEL: org
SLACK_ICON: https://avatars1.githubusercontent.com/u/30390575?s=48&v=4
SLACK_TITLE: New Core Version
SLACK_USERNAME: opBot

View File

@@ -9,57 +9,34 @@
[![chat](https://img.shields.io/badge/support-slack-01579b)](https://onepanel-ce.slack.com/join/shared_invite/zt-eyjnwec0-nLaHhjif9Y~gA05KuX6AUg#/)
[![license](https://img.shields.io/github/license/onepanelio/core?color=01579b)](https://opensource.org/licenses/Apache-2.0)
Production scale, Kubernetes-native vision AI platform, with fully integrated components for model building, automated labeling, data processing and model training pipelines.
Production scale, vision AI platform, with fully integrated components for model building, automated labeling, data processing and model training pipelines.
<img width="100%" src="img/onepanel.gif">
## Why Onepanel?
- End-to-end workflow and infrastructure automation for production scale vision AI
- Automatic resource management and on-demand scaling of CPU and GPU nodes
- Easily scale your data processing and training pipelines to multiple nodes
- Collaborate on all your deep learning tools and workflows through a unified web interface and SDKs
- End-to-end automation for production scale vision AI pipelines
- Best of breed, open source deep learning tools seamlessly integrated in one unified platform
- Infrastructure automation so you can easily scale your data processing and training pipelines to multiple nodes
- Customizable, reproducible and version controlled tooling and pipeline templates
- Scalability, flexibility and resiliency of Kubernetes without the deployment and configuration complexities
## Features
<table>
<tr>
<td width="50%" align="center">
<h3>Image and video annotation with automatic annotation</h3>
<img width="100%" src="img/auto-annotation.gif">
<p>
Annotate images and video with automatic annotation of bounding boxes and polygon masks, integrated with training pipelines to iteratively improve models for pre-annotation and inference
</p>
</td>
<td width="50%" align="center">
<h3>JupyterLab with TensorFlow, PyTorch and GPU support</h3>
<img width="100%" src="img/jupyterlab.gif">
<p>
JupyterLab configured with extensions for debugging, Git/GitHub, notebook diffing and TensorBoard and support for Conda, OpenCV, Tensorflow and PyTorch with GPU and <a href="https://github.com/onepanelio/templates/tree/master/workspaces/jupyterlab">much more</a>
</p>
</td>
</tr>
<tr>
<td width="50%" align="center">
<h3>Auto scaling, distributed and parallel data processing and training pipelines</h3>
<img width="100%" src="img/pipelines.gif">
<p>
Build fully reproducible, distributed and parallel data processing and training pipelines with real-time logs and output snapshots
</p>
</td>
<td width="50%" align="center">
<h3>Version controlled pipelines and environments as code</h3>
<img width="100%" src="img/tools.gif">
<p>
Bring your own IDEs, annotation tools and pipelines with a version controlled YAML and Docker based template engine
</p>
</td>
</tr>
</table>
- Annotate images and video with automatic annotation of bounding boxes and polygon masks, integrated with training pipelines to iteratively improve models for pre-annotation and inference.
- JupyterLab configured with extensions for debugging, Git/GitHub, notebook diffing and TensorBoard and support for Conda, OpenCV, Tensorflow and PyTorch with GPU and <a href="https://github.com/onepanelio/templates/tree/master/workspaces/jupyterlab">much more</a>.
- Build fully reproducible, distributed and parallel data processing and training pipelines with real-time logs and output snapshots.
- Bring your own IDEs, annotation tools and pipelines with a version controlled YAML and Docker based template engine.
- Track and visualize model metrics and experiments with TensorBoard or bring your own experiment tracking tools.
- Access and share tools like AirSim, Carla, Gazebo or OpenAI Gym through your browser with VNC enabled workspaces.
- Extend Onepanel with powerful REST APIs and SDKs to further automate your pipelines and environments.
- Workflows, environments and infrastructure are all defined as code and version controlled, making them reproducible and portable.
- Powered by Kubernetes so you can deploy anywhere Kubernetes can run.
- Extend Onepanel with powerful REST APIs and SDKs to further automate your workflows.
## Online demo
We have created an [online demo environment](https://onepanel.typeform.com/to/kQfDX5Vf?product=github) so that you can quickly try Onepanel.
Note that this is a shared demo environment with the following restrictions:
- Data is reset every few hours
- One type of node pool (machine type) with a limit of 5 concurrent nodes
- Certain actions may be restricted
## Quick start
See [quick start guide](https://docs.onepanel.ai/docs/getting-started/quickstart) to get started with the platform of your choice.
@@ -74,17 +51,17 @@ See [documentation](https://docs.onepanel.ai) to get started or for more detaile
To submit a feature request, report a bug or documentation issue, please open a GitHub [pull request](https://github.com/onepanelio/core/pulls) or [issue](https://github.com/onepanelio/core/issues).
For help, questions, release announcements and contribution discussions, join us on [Slack](https://join.slack.com/t/onepanel-ce/shared_invite/zt-eyjnwec0-nLaHhjif9Y~gA05KuX6AUg).
For help, questions, release announcements and contribution discussions, join us on [GitHub discussions](https://github.com/onepanelio/core/discussions).
## Contributing
Onepanel is modular and consists of the following repositories:
[Core API](https://github.com/onepanelio/core/) (this repository) - Code base for backend (Go)\
[Core UI](https://github.com/onepanelio/core-ui/) - Code base for UI (Angular + TypeScript)\
[CLI](https://github.com/onepanelio/cli/) - Code base for Go CLI for installation and management (Go)\
[Manifests](https://github.com/onepanelio/core-ui/) - Kustomize manifests used by CLI for installation and management (YAML)\
[Python SDK](https://github.com/onepanelio/python-sdk/) - Python SDK code and documentation\
[Backend](https://github.com/onepanelio/core/) (this repository) - Code base for backend (Go)\
[Frontend](https://github.com/onepanelio/core-ui/) - Code base for frontend (Angular + TypeScript)\
[CLI](https://github.com/onepanelio/cli/) - Code base for installation and management CLI (Go)\
[Manifests](https://github.com/onepanelio/core-ui/) - Kustomize manifests used by installation and management CLI (YAML)\
[Python SDK](https://github.com/onepanelio/python-sdk/) - Python SDK code and documentation (Python)\
[Templates](https://github.com/onepanelio/templates) - Various Workspace, Workflow, Task and Sidecar Templates\
[Documentation](https://github.com/onepanelio/core-docs/) - The repository for documentation site\
[API Documentation](https://github.com/onepanelio/core-api-docs/) - API documentation if you choose to use the API directly

View File

@@ -3,7 +3,7 @@
"info": {
"title": "Onepanel",
"description": "Onepanel API",
"version": "0.13.0",
"version": "0.14.0",
"contact": {
"name": "Onepanel project",
"url": "https://github.com/onepanelio/core"
@@ -77,7 +77,7 @@
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/TokenWrapper"
"$ref": "#/definitions/IsValidTokenRequest"
}
}
],
@@ -2967,11 +2967,28 @@
}
}
},
"IsValidTokenRequest": {
"type": "object",
"properties": {
"username": {
"type": "string"
},
"token": {
"type": "string"
}
}
},
"IsValidTokenResponse": {
"type": "object",
"properties": {
"domain": {
"type": "string"
},
"token": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
@@ -3394,14 +3411,6 @@
}
}
},
"TokenWrapper": {
"type": "object",
"properties": {
"token": {
"type": "string"
}
}
},
"UpdateSecretKeyValueResponse": {
"type": "object",
"properties": {

View File

@@ -204,65 +204,19 @@ func (x *IsAuthorizedResponse) GetAuthorized() bool {
return false
}
type TokenWrapper struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Token string `protobuf:"bytes,1,opt,name=token,proto3" json:"token,omitempty"`
}
func (x *TokenWrapper) Reset() {
*x = TokenWrapper{}
if protoimpl.UnsafeEnabled {
mi := &file_auth_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *TokenWrapper) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*TokenWrapper) ProtoMessage() {}
func (x *TokenWrapper) ProtoReflect() protoreflect.Message {
mi := &file_auth_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use TokenWrapper.ProtoReflect.Descriptor instead.
func (*TokenWrapper) Descriptor() ([]byte, []int) {
return file_auth_proto_rawDescGZIP(), []int{3}
}
func (x *TokenWrapper) GetToken() string {
if x != nil {
return x.Token
}
return ""
}
type IsValidTokenRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Token *TokenWrapper `protobuf:"bytes,1,opt,name=token,proto3" json:"token,omitempty"`
Username string `protobuf:"bytes,1,opt,name=username,proto3" json:"username,omitempty"`
Token string `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"`
}
func (x *IsValidTokenRequest) Reset() {
*x = IsValidTokenRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_auth_proto_msgTypes[4]
mi := &file_auth_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -275,7 +229,7 @@ func (x *IsValidTokenRequest) String() string {
func (*IsValidTokenRequest) ProtoMessage() {}
func (x *IsValidTokenRequest) ProtoReflect() protoreflect.Message {
mi := &file_auth_proto_msgTypes[4]
mi := &file_auth_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -288,14 +242,21 @@ func (x *IsValidTokenRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use IsValidTokenRequest.ProtoReflect.Descriptor instead.
func (*IsValidTokenRequest) Descriptor() ([]byte, []int) {
return file_auth_proto_rawDescGZIP(), []int{4}
return file_auth_proto_rawDescGZIP(), []int{3}
}
func (x *IsValidTokenRequest) GetToken() *TokenWrapper {
func (x *IsValidTokenRequest) GetUsername() string {
if x != nil {
return x.Username
}
return ""
}
func (x *IsValidTokenRequest) GetToken() string {
if x != nil {
return x.Token
}
return nil
return ""
}
type IsValidTokenResponse struct {
@@ -303,13 +264,15 @@ type IsValidTokenResponse struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Domain string `protobuf:"bytes,1,opt,name=domain,proto3" json:"domain,omitempty"`
Domain string `protobuf:"bytes,1,opt,name=domain,proto3" json:"domain,omitempty"`
Token string `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"`
Username string `protobuf:"bytes,3,opt,name=username,proto3" json:"username,omitempty"`
}
func (x *IsValidTokenResponse) Reset() {
*x = IsValidTokenResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_auth_proto_msgTypes[5]
mi := &file_auth_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -322,7 +285,7 @@ func (x *IsValidTokenResponse) String() string {
func (*IsValidTokenResponse) ProtoMessage() {}
func (x *IsValidTokenResponse) ProtoReflect() protoreflect.Message {
mi := &file_auth_proto_msgTypes[5]
mi := &file_auth_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -335,7 +298,7 @@ func (x *IsValidTokenResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use IsValidTokenResponse.ProtoReflect.Descriptor instead.
func (*IsValidTokenResponse) Descriptor() ([]byte, []int) {
return file_auth_proto_rawDescGZIP(), []int{5}
return file_auth_proto_rawDescGZIP(), []int{4}
}
func (x *IsValidTokenResponse) GetDomain() string {
@@ -345,6 +308,20 @@ func (x *IsValidTokenResponse) GetDomain() string {
return ""
}
func (x *IsValidTokenResponse) GetToken() string {
if x != nil {
return x.Token
}
return ""
}
func (x *IsValidTokenResponse) GetUsername() string {
if x != nil {
return x.Username
}
return ""
}
var File_auth_proto protoreflect.FileDescriptor
var file_auth_proto_rawDesc = []byte{
@@ -370,32 +347,33 @@ var file_auth_proto_rawDesc = []byte{
0x7a, 0x65, 0x64, 0x22, 0x36, 0x0a, 0x14, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69,
0x7a, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x61,
0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52,
0x0a, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x22, 0x24, 0x0a, 0x0c, 0x54,
0x6f, 0x6b, 0x65, 0x6e, 0x57, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x74,
0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65,
0x6e, 0x22, 0x3e, 0x0a, 0x13, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65,
0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65,
0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x54, 0x6f,
0x6b, 0x65, 0x6e, 0x57, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65,
0x6e, 0x22, 0x2e, 0x0a, 0x14, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65,
0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d,
0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69,
0x6e, 0x32, 0xea, 0x01, 0x0a, 0x0b, 0x41, 0x75, 0x74, 0x68, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63,
0x65, 0x12, 0x6c, 0x0a, 0x0c, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65,
0x6e, 0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54,
0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x61, 0x70,
0x69, 0x2e, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x27, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x21, 0x22, 0x18,
0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x61, 0x75,
0x74, 0x68, 0x2f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x3a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12,
0x6d, 0x0a, 0x0c, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x12,
0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a,
0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e,
0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x22, 0x28, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x22, 0x22, 0x12, 0x2f, 0x61,
0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x61, 0x75, 0x74, 0x68,
0x3a, 0x0c, 0x69, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x0a, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x22, 0x47, 0x0a, 0x13, 0x49,
0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14,
0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74,
0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x60, 0x0a, 0x14, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54,
0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06,
0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f,
0x6d, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20,
0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73,
0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73,
0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x32, 0xe6, 0x01, 0x0a, 0x0b, 0x41, 0x75, 0x74, 0x68, 0x53,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x68, 0x0a, 0x0c, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69,
0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x56,
0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x6f,
0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x23, 0x82, 0xd3, 0xe4,
0x93, 0x02, 0x1d, 0x22, 0x18, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74,
0x61, 0x31, 0x2f, 0x61, 0x75, 0x74, 0x68, 0x2f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x3a, 0x01, 0x2a,
0x12, 0x6d, 0x0a, 0x0c, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64,
0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69,
0x7a, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69,
0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x28, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x22, 0x22, 0x12, 0x2f,
0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x61, 0x75, 0x74,
0x68, 0x3a, 0x0c, 0x69, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -410,27 +388,25 @@ func file_auth_proto_rawDescGZIP() []byte {
return file_auth_proto_rawDescData
}
var file_auth_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
var file_auth_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
var file_auth_proto_goTypes = []interface{}{
(*IsAuthorized)(nil), // 0: api.IsAuthorized
(*IsAuthorizedRequest)(nil), // 1: api.IsAuthorizedRequest
(*IsAuthorizedResponse)(nil), // 2: api.IsAuthorizedResponse
(*TokenWrapper)(nil), // 3: api.TokenWrapper
(*IsValidTokenRequest)(nil), // 4: api.IsValidTokenRequest
(*IsValidTokenResponse)(nil), // 5: api.IsValidTokenResponse
(*IsValidTokenRequest)(nil), // 3: api.IsValidTokenRequest
(*IsValidTokenResponse)(nil), // 4: api.IsValidTokenResponse
}
var file_auth_proto_depIdxs = []int32{
0, // 0: api.IsAuthorizedRequest.isAuthorized:type_name -> api.IsAuthorized
3, // 1: api.IsValidTokenRequest.token:type_name -> api.TokenWrapper
4, // 2: api.AuthService.IsValidToken:input_type -> api.IsValidTokenRequest
1, // 3: api.AuthService.IsAuthorized:input_type -> api.IsAuthorizedRequest
5, // 4: api.AuthService.IsValidToken:output_type -> api.IsValidTokenResponse
2, // 5: api.AuthService.IsAuthorized:output_type -> api.IsAuthorizedResponse
4, // [4:6] is the sub-list for method output_type
2, // [2:4] is the sub-list for method input_type
2, // [2:2] is the sub-list for extension type_name
2, // [2:2] is the sub-list for extension extendee
0, // [0:2] is the sub-list for field type_name
3, // 1: api.AuthService.IsValidToken:input_type -> api.IsValidTokenRequest
1, // 2: api.AuthService.IsAuthorized:input_type -> api.IsAuthorizedRequest
4, // 3: api.AuthService.IsValidToken:output_type -> api.IsValidTokenResponse
2, // 4: api.AuthService.IsAuthorized:output_type -> api.IsAuthorizedResponse
3, // [3:5] is the sub-list for method output_type
1, // [1:3] is the sub-list for method input_type
1, // [1:1] is the sub-list for extension type_name
1, // [1:1] is the sub-list for extension extendee
0, // [0:1] is the sub-list for field type_name
}
func init() { file_auth_proto_init() }
@@ -476,18 +452,6 @@ func file_auth_proto_init() {
}
}
file_auth_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*TokenWrapper); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_auth_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*IsValidTokenRequest); i {
case 0:
return &v.state
@@ -499,7 +463,7 @@ func file_auth_proto_init() {
return nil
}
}
file_auth_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
file_auth_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*IsValidTokenResponse); i {
case 0:
return &v.state
@@ -518,7 +482,7 @@ func file_auth_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_auth_proto_rawDesc,
NumEnums: 0,
NumMessages: 6,
NumMessages: 5,
NumExtensions: 0,
NumServices: 1,
},

View File

@@ -39,7 +39,7 @@ func request_AuthService_IsValidToken_0(ctx context.Context, marshaler runtime.M
if berr != nil {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
}
if err := marshaler.NewDecoder(newReader()).Decode(&protoReq.Token); err != nil && err != io.EOF {
if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
}
@@ -56,7 +56,7 @@ func local_request_AuthService_IsValidToken_0(ctx context.Context, marshaler run
if berr != nil {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr)
}
if err := marshaler.NewDecoder(newReader()).Decode(&protoReq.Token); err != nil && err != io.EOF {
if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
}

View File

@@ -9,7 +9,7 @@ service AuthService {
rpc IsValidToken(IsValidTokenRequest) returns (IsValidTokenResponse) {
option (google.api.http) = {
post: "/apis/v1beta1/auth/token"
body: "token"
body: "*"
};
}
@@ -37,14 +37,13 @@ message IsAuthorizedResponse {
bool authorized = 1;
}
message TokenWrapper {
string token = 1;
}
message IsValidTokenRequest {
TokenWrapper token = 1;
string username = 1;
string token = 2;
}
message IsValidTokenResponse {
string domain = 1;
string token = 2;
string username = 3;
}

View File

@@ -0,0 +1,147 @@
# Workspace arguments
arguments:
parameters:
- name: sync-directory
displayName: Directory to sync raw input and training output
value: workflow-data
hint: Location to sync raw input, models and checkpoints from default object storage. Note that this will be relative to the current namespace.
containers:
- name: cvat-db
image: postgres:10-alpine
env:
- name: POSTGRES_USER
value: root
- name: POSTGRES_DB
value: cvat
- name: POSTGRES_HOST_AUTH_METHOD
value: trust
- name: PGDATA
value: /var/lib/psql/data
ports:
- containerPort: 5432
name: tcp
volumeMounts:
- name: db
mountPath: /var/lib/psql
- name: cvat-redis
image: redis:4.0-alpine
ports:
- containerPort: 6379
name: tcp
- name: cvat
image: onepanel/cvat:0.14.0_cvat.1.0.0
env:
- name: DJANGO_MODWSGI_EXTRA_ARGS
value: ""
- name: ALLOWED_HOSTS
value: '*'
- name: CVAT_REDIS_HOST
value: localhost
- name: CVAT_POSTGRES_HOST
value: localhost
- name: CVAT_SHARE_URL
value: /home/django/data
- name: ONEPANEL_SYNC_DIRECTORY
value: '{{workspace.parameters.sync-directory}}'
- name: NVIDIA_VISIBLE_DEVICES
value: all
- name: NVIDIA_DRIVER_CAPABILITIES
value: compute,utility
- name: NVIDIA_REQUIRE_CUDA
value: "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
ports:
- containerPort: 8080
name: http
volumeMounts:
- name: data
mountPath: /home/django/data
- name: keys
mountPath: /home/django/keys
- name: logs
mountPath: /home/django/logs
- name: models
mountPath: /home/django/models
- name: share
mountPath: /home/django/share
- name: sys-namespace-config
mountPath: /etc/onepanel
readOnly: true
- name: cvat-ui
image: onepanel/cvat-ui:0.14.0_cvat.1.0.0
ports:
- containerPort: 80
name: http
# You can add multiple FileSyncer sidecar containers if needed
- name: filesyncer
image: onepanel/filesyncer:{{.ArtifactRepositoryType}}
imagePullPolicy: Always
args:
- download
- -server-prefix=/sys/filesyncer
env:
- name: FS_PATH
value: /mnt/share
- name: FS_PREFIX
value: '{{workflow.namespace}}/{{workspace.parameters.sync-directory}}'
volumeMounts:
- name: share
mountPath: /mnt/share
- name: sys-namespace-config
mountPath: /etc/onepanel
readOnly: true
ports:
- name: cvat-ui
port: 80
protocol: TCP
targetPort: 80
- name: cvat
port: 8080
protocol: TCP
targetPort: 8080
- name: fs
port: 8888
protocol: TCP
targetPort: 8888
routes:
- match:
- uri:
prefix: /sys/filesyncer
route:
- destination:
port:
number: 8888
- match:
- uri:
regex: /api/.*|/git/.*|/tensorflow/.*|/onepanelio/.*|/tracking/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.*
- queryParams:
id:
regex: \d+.*
route:
- destination:
port:
number: 8080
- match:
- uri:
prefix: /
route:
- destination:
port:
number: 80
# DAG Workflow to be executed once a Workspace action completes (optional)
# Uncomment the lines below if you want to send Slack notifications
#postExecutionWorkflow:
# entrypoint: main
# templates:
# - name: main
# dag:
# tasks:
# - name: slack-notify
# template: slack-notify
# - name: slack-notify
# container:
# image: technosophos/slack-notify
# args:
# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify
# command:
# - sh
# - -c

View File

@@ -0,0 +1,25 @@
package migration
import (
"database/sql"
"github.com/pressly/goose"
)
func initialize20201016170415() {
if _, ok := initializedMigrations[20201016170415]; !ok {
goose.AddMigration(Up20201016170415, Down20201016170415)
initializedMigrations[20201016170415] = true
}
}
// Up20201016170415 updates cvat to a new version
func Up20201016170415(tx *sql.Tx) error {
// This code is executed when the migration is applied.
return updateWorkspaceTemplateManifest("cvat_20201016170415.yaml", cvatTemplateName)
}
// Down20201016170415 does nothing
func Down20201016170415(tx *sql.Tx) error {
// This code is executed when the migration is rolled back.
return nil
}

View File

@@ -5,7 +5,10 @@ import (
sq "github.com/Masterminds/squirrel"
"github.com/jmoiron/sqlx"
v1 "github.com/onepanelio/core/pkg"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
)
@@ -60,6 +63,7 @@ func Initialize() {
initialize20200929144301()
initialize20200929153931()
initialize20201001070806()
initialize20201016170415()
if err := client.DB.Close(); err != nil {
log.Printf("[error] closing db %v", err)
@@ -128,3 +132,17 @@ func ReplaceArtifactRepositoryType(client *v1.Client, namespace *v1.Namespace, w
return nil
}
// readDataFile returns the contents of a file in the db/data/{name} directory
func readDataFile(name string) (string, error) {
curDir, err := os.Getwd()
if err != nil {
return "", err
}
data, err := ioutil.ReadFile(filepath.Join(curDir, "db", "data", name))
if err != nil {
return "", err
}
return string(data), nil
}

49
db/go/util.go Normal file
View File

@@ -0,0 +1,49 @@
package migration
import (
v1 "github.com/onepanelio/core/pkg"
uid2 "github.com/onepanelio/core/pkg/util/uid"
)
// updateWorkspaceTemplateManifest will update the workspace template given by {{templateName}} with the contents
// given by {{filename}}
// It will do so for all namespaces.
func updateWorkspaceTemplateManifest(filename, templateName string) error {
client, err := getClient()
if err != nil {
return err
}
defer client.DB.Close()
namespaces, err := client.ListOnepanelEnabledNamespaces()
if err != nil {
return err
}
newManifest, err := readDataFile(filename)
if err != nil {
return err
}
uid, err := uid2.GenerateUID(templateName, 30)
if err != nil {
return err
}
for _, namespace := range namespaces {
workspaceTemplate := &v1.WorkspaceTemplate{
UID: uid,
Name: templateName,
Manifest: newManifest,
}
err = ReplaceArtifactRepositoryType(client, namespace, nil, workspaceTemplate)
if err != nil {
return err
}
if _, err := client.UpdateWorkspaceTemplateManifest(namespace.Name, uid, workspaceTemplate.Manifest); err != nil {
return err
}
}
return nil
}

3
go.mod
View File

@@ -43,4 +43,5 @@ require (
k8s.io/apimachinery v0.16.7-beta.0
k8s.io/client-go v0.16.4
sigs.k8s.io/yaml v1.2.0
)
github.com/dgrijalva/jwt-go v3.2.0+incompatible
)

BIN
img/onepanel.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.7 MiB

View File

@@ -4,6 +4,7 @@ import (
"context"
"flag"
"fmt"
migrations "github.com/onepanelio/core/db/go"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
corev1 "k8s.io/api/core/v1"
@@ -25,7 +26,6 @@ import (
"github.com/grpc-ecosystem/grpc-gateway/runtime"
"github.com/jmoiron/sqlx"
"github.com/onepanelio/core/api"
migrations "github.com/onepanelio/core/db/go"
v1 "github.com/onepanelio/core/pkg"
"github.com/onepanelio/core/pkg/util/env"
"github.com/onepanelio/core/server"

View File

@@ -4,6 +4,7 @@ import (
"fmt"
sq "github.com/Masterminds/squirrel"
argoprojv1alpha1 "github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
"github.com/jmoiron/sqlx"
"github.com/onepanelio/core/pkg/util/gcs"
"github.com/onepanelio/core/pkg/util/router"
"github.com/onepanelio/core/pkg/util/s3"
@@ -38,6 +39,37 @@ func NewConfig() (config *Config) {
return
}
// GetDefaultClient loads a default k8s client
func GetDefaultClient() (*Client, error) {
kubeConfig := NewConfig()
client, err := NewClient(kubeConfig, nil, nil)
if err != nil {
return nil, err
}
config, err := client.GetSystemConfig()
if err != nil {
return nil, err
}
dbDriverName, dbDataSourceName := config.DatabaseConnection()
client.DB = NewDB(sqlx.MustConnect(dbDriverName, dbDataSourceName))
return client, nil
}
// GetDefaultClientWithDB loads a default k8s client with an existing DB
func GetDefaultClientWithDB(db *DB) (*Client, error) {
kubeConfig := NewConfig()
client, err := NewClient(kubeConfig, nil, nil)
if err != nil {
return nil, err
}
client.DB = db
return client, nil
}
// NewClient creates a client to interact with the Onepanel system.
// It includes access to the database, kubernetes, argo, and configuration.
func NewClient(config *Config, db *DB, systemConfig SystemConfig) (client *Client, err error) {

View File

@@ -38,6 +38,12 @@ func NewSystemConfig(configMap *ConfigMap, secret *Secret) (config SystemConfig,
}
config["databasePassword"] = string(databasePassword)
hmac, err := base64.StdEncoding.DecodeString(secret.Data["hmac"])
if err != nil {
return
}
config["hmac"] = string(hmac)
return
}
@@ -183,6 +189,16 @@ func (s SystemConfig) UpdateNodePoolOptions(parameters []Parameter) ([]Parameter
return result, nil
}
// HMACKey gets the HMAC value, or nil.
func (s SystemConfig) HMACKey() []byte {
hmac := s.GetValue("hmac")
if hmac == nil {
return []byte{}
}
return []byte(*hmac)
}
// ArtifactRepositoryS3Provider is meant to be used
// by the CLI. CLI will marshal this struct into the correct
// YAML structure for k8s configmap / secret.

View File

@@ -1340,7 +1340,7 @@ func (c *Client) ListFiles(namespace, key string) (files []*File, err error) {
doneCh := make(chan struct{})
defer close(doneCh)
for objInfo := range s3Client.ListObjectsV2(config.ArtifactRepository.S3.Bucket, key, false, doneCh) {
for objInfo := range s3Client.ListObjects(config.ArtifactRepository.S3.Bucket, key, false, doneCh) {
if objInfo.Key == key {
continue
}
@@ -1851,14 +1851,14 @@ func workflowExecutionsSelectBuilderNoColumns(namespace, workflowTemplateUID, wo
func workflowExecutionsSelectBuilder(namespace, workflowTemplateUID, workflowTemplateVersion string, includeSystem bool) sq.SelectBuilder {
sb := workflowExecutionsSelectBuilderNoColumns(namespace, workflowTemplateUID, workflowTemplateVersion, includeSystem)
sb = sb.Columns(getWorkflowExecutionColumns("we", "")...).
sb = sb.Columns(getWorkflowExecutionColumns("we")...).
Columns(`wtv.version "workflow_template.version"`, `wtv.created_at "workflow_template.created_at"`, `wt.name "workflow_template.name"`, `wt.uid "workflow_template.uid"`)
return sb
}
func (c *Client) getWorkflowExecutionAndTemplate(namespace string, uid string) (workflow *WorkflowExecution, err error) {
sb := sb.Select(getWorkflowExecutionColumns("we", "")...).
sb := sb.Select(getWorkflowExecutionColumns("we")...).
Columns(getWorkflowTemplateColumns("wt", "workflow_template")...).
Columns(`wtv.manifest "workflow_template.manifest"`, `wtv.version "workflow_template.version"`).
From("workflow_executions we").

View File

@@ -401,7 +401,7 @@ func createStatefulSetManifest(spec *WorkspaceSpec, config map[string]string, se
env.PrependEnvVarToContainer(container, "ONEPANEL_DOMAIN", config["ONEPANEL_DOMAIN"])
env.PrependEnvVarToContainer(container, "ONEPANEL_PROVIDER", config["ONEPANEL_PROVIDER"])
env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_NAMESPACE", "{{workflow.namespace}}")
env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_UID", "{{workflow.parameters.sys-name}}")
env.PrependEnvVarToContainer(container, "ONEPANEL_RESOURCE_UID", "{{workflow.parameters.sys-uid}}")
for _, service := range services {
envName := fmt.Sprintf("ONEPANEL_SERVICES_%v_API_URL", strings.ToUpper(service.Name))

View File

@@ -2,9 +2,13 @@ package auth
import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"github.com/onepanelio/core/api"
"github.com/onepanelio/core/pkg/util"
v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
"net/http"
"strings"
@@ -36,6 +40,9 @@ func getBearerToken(ctx context.Context) (*string, bool) {
return nil, false
}
t = strings.ReplaceAll(t, prefix, "")
if t == "null" {
return nil, false
}
return &t, true
}
@@ -64,7 +71,12 @@ func getClient(ctx context.Context, kubeConfig *v1.Config, db *v1.DB, sysConfig
return nil, status.Error(codes.Unauthenticated, `Missing or invalid "authorization" header.`)
}
if sysConfig["token"] != *bearerToken {
sysConfig["token"] = *bearerToken
}
kubeConfig.BearerToken = *bearerToken
client, err := v1.NewClient(kubeConfig, db, sysConfig)
if err != nil {
return nil, err
@@ -98,6 +110,43 @@ func IsAuthorized(c *v1.Client, namespace, verb, group, resource, name string) (
return
}
func verifyLogin(client *v1.Client, tokenRequest *api.IsValidTokenRequest) (rawToken string, err error) {
accountsList, err := client.CoreV1().ServiceAccounts("onepanel").List(v1.ListOptions{})
if err != nil {
return "", err
}
authTokenSecretName := ""
for _, serviceAccount := range accountsList.Items {
if serviceAccount.Name != tokenRequest.Username {
continue
}
for _, secret := range serviceAccount.Secrets {
if strings.Contains(secret.Name, "-token-") {
authTokenSecretName = secret.Name
break
}
}
}
if authTokenSecretName == "" {
return "", util.NewUserError(codes.InvalidArgument, fmt.Sprintf("unknown service account '%v'", tokenRequest.Username))
}
secret, err := client.CoreV1().Secrets("onepanel").Get(authTokenSecretName, v12.GetOptions{})
if err != nil {
return "", err
}
currentTokenBytes := md5.Sum(secret.Data["token"])
currentTokenString := hex.EncodeToString(currentTokenBytes[:])
if tokenRequest.Token != fmt.Sprintf("%s", currentTokenString) {
return "", util.NewUserError(codes.InvalidArgument, "token doesn't match what's on record")
}
return string(secret.Data["token"]), nil
}
// UnaryInterceptor performs authentication checks.
// The two main cases are:
// 1. Is the token valid? This is used for logging in.
@@ -113,10 +162,25 @@ func UnaryInterceptor(kubeConfig *v1.Config, db *v1.DB, sysConfig v1.SystemConfi
tokenRequest, ok := req.(*api.IsValidTokenRequest)
if !ok {
return resp, errors.New("IsValidToken does not have correct request type")
return resp, errors.New("LogInRequest does not have correct request type")
}
md.Set("authorization", tokenRequest.Token.Token)
defaultClient, err := v1.GetDefaultClientWithDB(db)
if err != nil {
return nil, err
}
rawToken, err := verifyLogin(defaultClient, tokenRequest)
if err != nil {
return nil, err
}
sysConfig, err := defaultClient.GetSystemConfig()
if err != nil {
return nil, err
}
md.Set("onepanel-auth-token", rawToken)
ctx, err = getClient(ctx, kubeConfig, db, sysConfig)
if err != nil {

View File

@@ -66,8 +66,10 @@ func (a *AuthServer) IsValidToken(ctx context.Context, req *api.IsValidTokenRequ
if err != nil {
return
}
res = &api.IsValidTokenResponse{}
res.Domain = config["ONEPANEL_DOMAIN"]
res = &api.IsValidTokenResponse{
Domain: config["ONEPANEL_DOMAIN"],
Token: config["token"],
}
return res, nil
}

View File

@@ -7,21 +7,22 @@ import (
"github.com/onepanelio/core/server/auth"
)
func resourceIdentifierToArgoResource(identifier string) string {
func getGroupAndResourceByIdentifier(identifier string) (group, resource string) {
group = "argoproj.io"
switch identifier {
case v1.TypeWorkflowTemplate:
return "workflowtemplates"
return group, "workflowtemplates"
case v1.TypeWorkflowTemplateVersion:
return "workflowtemplates"
return group, "workflowtemplates"
case v1.TypeWorkflowExecution:
return "workflows"
return group, "workflows"
case v1.TypeCronWorkflow:
return "cronworkflows"
return group, "cronworkflows"
case v1.TypeWorkspace:
return "statefulset"
return "onepanel.io", "workspaces"
}
return ""
return "", ""
}
func mapLabelsToKeyValue(labels []*v1.Label) []*api.KeyValue {
@@ -54,10 +55,10 @@ func NewLabelServer() *LabelServer {
}
func (s *LabelServer) GetLabels(ctx context.Context, req *api.GetLabelsRequest) (*api.GetLabelsResponse, error) {
argoResource := resourceIdentifierToArgoResource(req.Resource)
group, resource := getGroupAndResourceByIdentifier(req.Resource)
client := getClient(ctx)
allowed, err := auth.IsAuthorized(client, req.Namespace, "get", "argoproj.io", argoResource, "")
allowed, err := auth.IsAuthorized(client, req.Namespace, "get", group, resource, "")
if err != nil || !allowed {
return nil, err
}
@@ -73,10 +74,10 @@ func (s *LabelServer) GetLabels(ctx context.Context, req *api.GetLabelsRequest)
}
func (s *LabelServer) AddLabels(ctx context.Context, req *api.AddLabelsRequest) (*api.GetLabelsResponse, error) {
argoResource := resourceIdentifierToArgoResource(req.Resource)
group, resource := getGroupAndResourceByIdentifier(req.Resource)
client := getClient(ctx)
allowed, err := auth.IsAuthorized(client, req.Namespace, "create", "argoproj.io", argoResource, "")
allowed, err := auth.IsAuthorized(client, req.Namespace, "create", group, resource, "")
if err != nil || !allowed {
return nil, err
}
@@ -97,10 +98,10 @@ func (s *LabelServer) AddLabels(ctx context.Context, req *api.AddLabelsRequest)
}
func (s *LabelServer) ReplaceLabels(ctx context.Context, req *api.ReplaceLabelsRequest) (*api.GetLabelsResponse, error) {
argoResource := resourceIdentifierToArgoResource(req.Resource)
group, resource := getGroupAndResourceByIdentifier(req.Resource)
client := getClient(ctx)
allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "argoproj.io", argoResource, "")
allowed, err := auth.IsAuthorized(client, req.Namespace, "update", group, resource, "")
if err != nil || !allowed {
return nil, err
}
@@ -121,11 +122,11 @@ func (s *LabelServer) ReplaceLabels(ctx context.Context, req *api.ReplaceLabelsR
}
func (s *LabelServer) DeleteLabel(ctx context.Context, req *api.DeleteLabelRequest) (*api.GetLabelsResponse, error) {
argoResource := resourceIdentifierToArgoResource(req.Resource)
group, resource := getGroupAndResourceByIdentifier(req.Resource)
client := getClient(ctx)
// update verb here since we are not deleting the resource, but labels
allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "argoproj.io", argoResource, "")
allowed, err := auth.IsAuthorized(client, req.Namespace, "update", group, resource, "")
if err != nil || !allowed {
return nil, err
}

View File

@@ -363,7 +363,7 @@ func (s *WorkspaceServer) RetryLastWorkspaceAction(ctx context.Context, req *api
func (s *WorkspaceServer) GetWorkspaceStatisticsForNamespace(ctx context.Context, req *api.GetWorkspaceStatisticsForNamespaceRequest) (*api.GetWorkspaceStatisticsForNamespaceResponse, error) {
client := getClient(ctx)
allowed, err := auth.IsAuthorized(client, req.Namespace, "list", "argoproj.io", "workspaces", "")
allowed, err := auth.IsAuthorized(client, req.Namespace, "list", "onepanel.io", "workspaces", "")
if err != nil || !allowed {
return nil, err
}