diff --git a/.github/workflows/run_unit_tests.yaml b/.github/workflows/run_unit_tests.yaml new file mode 100644 index 0000000..bc00587 --- /dev/null +++ b/.github/workflows/run_unit_tests.yaml @@ -0,0 +1,26 @@ +name: Run Unit Tests +on: + push: + branches: + - test/docker.database +jobs: + test-code-job: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:12.3 + env: + POSTGRES_DB: onepanel + POSTGRES_USER: admin + POSTGRES_PASSWORD: tester + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@master + - name: Run testing code + uses: cedrickring/golang-action@1.5.2 + with: + args: go test github.com/onepanelio/core/pkg -db=postgres \ No newline at end of file diff --git a/Makefile b/Makefile index 3ccc46c..b5dd32d 100644 --- a/Makefile +++ b/Makefile @@ -34,3 +34,8 @@ docker-push: docker push onepanel/core:$(COMMIT_HASH) docker: docker-build docker-push + +run-tests: + docker run --rm --name test-onepanel-postgres -p 5432:5432 -e POSTGRES_USER=admin -e POSTGRES_PASSWORD=tester -e POSTGRES_DB=onepanel -d postgres:12.3 + go test github.com/onepanelio/core/pkg -count=1 ||: + docker stop test-onepanel-postgres \ No newline at end of file diff --git a/api/api.swagger.json b/api/api.swagger.json index 044cb16..05e20d2 100644 --- a/api/api.swagger.json +++ b/api/api.swagger.json @@ -1707,57 +1707,6 @@ ] } }, - "/apis/v1beta1/{namespace}/workflow_templates/{workflowTemplate.uid}/versions/{workflowTemplate.version}": { - "put": { - "operationId": "UpdateWorkflowTemplateVersion", - "responses": { - "200": { - "description": "A successful response.", - "schema": { - "$ref": "#/definitions/WorkflowTemplate" - } - }, - "default": { - "description": "An unexpected error response", - "schema": { - "$ref": "#/definitions/grpc.gateway.runtime.Error" - } - } - }, - "parameters": [ - { - "name": "namespace", - "in": "path", - "required": true, - "type": "string" - }, - { - "name": "workflowTemplate.uid", - "in": "path", - "required": true, - "type": "string" - }, - { - "name": "workflowTemplate.version", - "in": "path", - "required": true, - "type": "string", - "format": "int64" - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/WorkflowTemplate" - } - } - ], - "tags": [ - "WorkflowTemplateService" - ] - } - }, "/apis/v1beta1/{namespace}/workspace_templates": { "get": { "operationId": "ListWorkspaceTemplates", diff --git a/api/workflow_template.pb.go b/api/workflow_template.pb.go index 6aa1297..ecdbc15 100644 --- a/api/workflow_template.pb.go +++ b/api/workflow_template.pb.go @@ -1093,7 +1093,7 @@ var file_workflow_template_proto_rawDesc = []byte{ 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x32, 0xa3, 0x0c, 0x0a, 0x17, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x32, 0xbb, 0x0a, 0x0a, 0x17, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x9b, 0x01, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x22, 0x2e, 0x61, @@ -1105,94 +1105,79 @@ var file_workflow_template_proto_rawDesc = []byte{ 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x3a, 0x10, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, - 0xe5, 0x01, 0x0a, 0x1d, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0xc2, 0x01, 0x0a, 0x1d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x22, 0x81, 0x01, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x7b, 0x1a, 0x67, 0x2f, 0x61, - 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, - 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x77, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x75, 0x69, 0x64, - 0x7d, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x7b, 0x77, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x7d, 0x3a, 0x10, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0xc2, 0x01, 0x0a, 0x1d, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x22, 0x66, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x60, 0x22, 0x4c, 0x2f, 0x61, - 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, - 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x77, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x75, 0x69, 0x64, - 0x7d, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x3a, 0x10, 0x77, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0xd3, 0x01, 0x0a, - 0x13, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x57, 0x6f, + 0x6e, 0x12, 0x22, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x57, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x83, 0x01, 0x82, - 0xd3, 0xe4, 0x93, 0x02, 0x7d, 0x12, 0x32, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, - 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, - 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x5a, 0x47, 0x12, 0x45, 0x2f, 0x61, 0x70, - 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, - 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x7b, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x7d, 0x12, 0xb8, 0x01, 0x0a, 0x1c, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x28, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, - 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x43, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x3d, - 0x12, 0x3b, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x7b, - 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x94, 0x01, - 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x12, 0x21, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x34, - 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2e, 0x12, 0x2c, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, - 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x73, 0x12, 0xe9, 0x01, 0x0a, 0x15, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x57, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x21, - 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, - 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x95, 0x01, 0x82, 0xd3, 0xe4, 0x93, 0x02, - 0x8e, 0x01, 0x12, 0x3f, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, - 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, - 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x2f, 0x7b, 0x6e, 0x61, - 0x6d, 0x65, 0x7d, 0x5a, 0x4b, 0x12, 0x49, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, - 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, - 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x2f, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x2f, 0x7b, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x7d, - 0x12, 0xa8, 0x01, 0x0a, 0x17, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x57, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, - 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x57, - 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x42, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x3c, 0x1a, - 0x3a, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x75, - 0x69, 0x64, 0x7d, 0x2f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x22, 0x66, 0x82, 0xd3, + 0xe4, 0x93, 0x02, 0x60, 0x22, 0x4c, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, + 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, + 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x73, 0x2f, 0x7b, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x2e, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x73, 0x3a, 0x10, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x12, 0xd3, 0x01, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, + 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, + 0x61, 0x70, 0x69, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x22, 0x83, 0x01, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x7d, 0x12, 0x32, 0x2f, + 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, + 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x75, 0x69, 0x64, + 0x7d, 0x5a, 0x47, 0x12, 0x45, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, + 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, + 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x73, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, + 0x2f, 0x7b, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x7d, 0x12, 0xb8, 0x01, 0x0a, 0x1c, 0x4c, + 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x28, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x43, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x3d, 0x12, 0x3b, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, + 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x94, 0x01, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, + 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x12, + 0x21, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, + 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x34, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2e, 0x12, 0x2c, + 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x12, 0xe9, 0x01, 0x0a, + 0x15, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x21, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x6c, 0x6f, + 0x6e, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x22, 0x95, 0x01, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x8e, 0x01, 0x12, 0x3f, 0x2f, 0x61, 0x70, 0x69, + 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x63, + 0x6c, 0x6f, 0x6e, 0x65, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x5a, 0x4b, 0x12, 0x49, 0x2f, + 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, + 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x75, 0x69, 0x64, + 0x7d, 0x2f, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x2f, 0x7b, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x7d, 0x12, 0xa8, 0x01, 0x0a, 0x17, 0x41, 0x72, 0x63, + 0x68, 0x69, 0x76, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, + 0x76, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x42, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x3c, 0x1a, 0x3a, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, + 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x7d, 0x2f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x73, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x61, 0x72, 0x63, 0x68, + 0x69, 0x76, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1235,23 +1220,21 @@ var file_workflow_template_proto_depIdxs = []int32{ 10, // 6: api.WorkflowTemplate.stats:type_name -> api.WorkflowExecutionStatisticReport 11, // 7: api.WorkflowTemplate.cronStats:type_name -> api.CronWorkflowStatisticsReport 0, // 8: api.WorkflowTemplateService.CreateWorkflowTemplate:input_type -> api.CreateWorkflowTemplateRequest - 1, // 9: api.WorkflowTemplateService.UpdateWorkflowTemplateVersion:input_type -> api.UpdateWorkflowTemplateVersionRequest - 0, // 10: api.WorkflowTemplateService.CreateWorkflowTemplateVersion:input_type -> api.CreateWorkflowTemplateRequest - 2, // 11: api.WorkflowTemplateService.GetWorkflowTemplate:input_type -> api.GetWorkflowTemplateRequest - 4, // 12: api.WorkflowTemplateService.ListWorkflowTemplateVersions:input_type -> api.ListWorkflowTemplateVersionsRequest - 6, // 13: api.WorkflowTemplateService.ListWorkflowTemplates:input_type -> api.ListWorkflowTemplatesRequest - 3, // 14: api.WorkflowTemplateService.CloneWorkflowTemplate:input_type -> api.CloneWorkflowTemplateRequest - 8, // 15: api.WorkflowTemplateService.ArchiveWorkflowTemplate:input_type -> api.ArchiveWorkflowTemplateRequest - 12, // 16: api.WorkflowTemplateService.CreateWorkflowTemplate:output_type -> api.WorkflowTemplate - 12, // 17: api.WorkflowTemplateService.UpdateWorkflowTemplateVersion:output_type -> api.WorkflowTemplate - 12, // 18: api.WorkflowTemplateService.CreateWorkflowTemplateVersion:output_type -> api.WorkflowTemplate - 12, // 19: api.WorkflowTemplateService.GetWorkflowTemplate:output_type -> api.WorkflowTemplate - 5, // 20: api.WorkflowTemplateService.ListWorkflowTemplateVersions:output_type -> api.ListWorkflowTemplateVersionsResponse - 7, // 21: api.WorkflowTemplateService.ListWorkflowTemplates:output_type -> api.ListWorkflowTemplatesResponse - 12, // 22: api.WorkflowTemplateService.CloneWorkflowTemplate:output_type -> api.WorkflowTemplate - 9, // 23: api.WorkflowTemplateService.ArchiveWorkflowTemplate:output_type -> api.ArchiveWorkflowTemplateResponse - 16, // [16:24] is the sub-list for method output_type - 8, // [8:16] is the sub-list for method input_type + 0, // 9: api.WorkflowTemplateService.CreateWorkflowTemplateVersion:input_type -> api.CreateWorkflowTemplateRequest + 2, // 10: api.WorkflowTemplateService.GetWorkflowTemplate:input_type -> api.GetWorkflowTemplateRequest + 4, // 11: api.WorkflowTemplateService.ListWorkflowTemplateVersions:input_type -> api.ListWorkflowTemplateVersionsRequest + 6, // 12: api.WorkflowTemplateService.ListWorkflowTemplates:input_type -> api.ListWorkflowTemplatesRequest + 3, // 13: api.WorkflowTemplateService.CloneWorkflowTemplate:input_type -> api.CloneWorkflowTemplateRequest + 8, // 14: api.WorkflowTemplateService.ArchiveWorkflowTemplate:input_type -> api.ArchiveWorkflowTemplateRequest + 12, // 15: api.WorkflowTemplateService.CreateWorkflowTemplate:output_type -> api.WorkflowTemplate + 12, // 16: api.WorkflowTemplateService.CreateWorkflowTemplateVersion:output_type -> api.WorkflowTemplate + 12, // 17: api.WorkflowTemplateService.GetWorkflowTemplate:output_type -> api.WorkflowTemplate + 5, // 18: api.WorkflowTemplateService.ListWorkflowTemplateVersions:output_type -> api.ListWorkflowTemplateVersionsResponse + 7, // 19: api.WorkflowTemplateService.ListWorkflowTemplates:output_type -> api.ListWorkflowTemplatesResponse + 12, // 20: api.WorkflowTemplateService.CloneWorkflowTemplate:output_type -> api.WorkflowTemplate + 9, // 21: api.WorkflowTemplateService.ArchiveWorkflowTemplate:output_type -> api.ArchiveWorkflowTemplateResponse + 15, // [15:22] is the sub-list for method output_type + 8, // [8:15] is the sub-list for method input_type 8, // [8:8] is the sub-list for extension type_name 8, // [8:8] is the sub-list for extension extendee 0, // [0:8] is the sub-list for field type_name @@ -1466,7 +1449,6 @@ const _ = grpc.SupportPackageIsVersion6 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type WorkflowTemplateServiceClient interface { CreateWorkflowTemplate(ctx context.Context, in *CreateWorkflowTemplateRequest, opts ...grpc.CallOption) (*WorkflowTemplate, error) - UpdateWorkflowTemplateVersion(ctx context.Context, in *UpdateWorkflowTemplateVersionRequest, opts ...grpc.CallOption) (*WorkflowTemplate, error) CreateWorkflowTemplateVersion(ctx context.Context, in *CreateWorkflowTemplateRequest, opts ...grpc.CallOption) (*WorkflowTemplate, error) GetWorkflowTemplate(ctx context.Context, in *GetWorkflowTemplateRequest, opts ...grpc.CallOption) (*WorkflowTemplate, error) ListWorkflowTemplateVersions(ctx context.Context, in *ListWorkflowTemplateVersionsRequest, opts ...grpc.CallOption) (*ListWorkflowTemplateVersionsResponse, error) @@ -1492,15 +1474,6 @@ func (c *workflowTemplateServiceClient) CreateWorkflowTemplate(ctx context.Conte return out, nil } -func (c *workflowTemplateServiceClient) UpdateWorkflowTemplateVersion(ctx context.Context, in *UpdateWorkflowTemplateVersionRequest, opts ...grpc.CallOption) (*WorkflowTemplate, error) { - out := new(WorkflowTemplate) - err := c.cc.Invoke(ctx, "/api.WorkflowTemplateService/UpdateWorkflowTemplateVersion", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *workflowTemplateServiceClient) CreateWorkflowTemplateVersion(ctx context.Context, in *CreateWorkflowTemplateRequest, opts ...grpc.CallOption) (*WorkflowTemplate, error) { out := new(WorkflowTemplate) err := c.cc.Invoke(ctx, "/api.WorkflowTemplateService/CreateWorkflowTemplateVersion", in, out, opts...) @@ -1558,7 +1531,6 @@ func (c *workflowTemplateServiceClient) ArchiveWorkflowTemplate(ctx context.Cont // WorkflowTemplateServiceServer is the server API for WorkflowTemplateService service. type WorkflowTemplateServiceServer interface { CreateWorkflowTemplate(context.Context, *CreateWorkflowTemplateRequest) (*WorkflowTemplate, error) - UpdateWorkflowTemplateVersion(context.Context, *UpdateWorkflowTemplateVersionRequest) (*WorkflowTemplate, error) CreateWorkflowTemplateVersion(context.Context, *CreateWorkflowTemplateRequest) (*WorkflowTemplate, error) GetWorkflowTemplate(context.Context, *GetWorkflowTemplateRequest) (*WorkflowTemplate, error) ListWorkflowTemplateVersions(context.Context, *ListWorkflowTemplateVersionsRequest) (*ListWorkflowTemplateVersionsResponse, error) @@ -1574,9 +1546,6 @@ type UnimplementedWorkflowTemplateServiceServer struct { func (*UnimplementedWorkflowTemplateServiceServer) CreateWorkflowTemplate(context.Context, *CreateWorkflowTemplateRequest) (*WorkflowTemplate, error) { return nil, status.Errorf(codes.Unimplemented, "method CreateWorkflowTemplate not implemented") } -func (*UnimplementedWorkflowTemplateServiceServer) UpdateWorkflowTemplateVersion(context.Context, *UpdateWorkflowTemplateVersionRequest) (*WorkflowTemplate, error) { - return nil, status.Errorf(codes.Unimplemented, "method UpdateWorkflowTemplateVersion not implemented") -} func (*UnimplementedWorkflowTemplateServiceServer) CreateWorkflowTemplateVersion(context.Context, *CreateWorkflowTemplateRequest) (*WorkflowTemplate, error) { return nil, status.Errorf(codes.Unimplemented, "method CreateWorkflowTemplateVersion not implemented") } @@ -1618,24 +1587,6 @@ func _WorkflowTemplateService_CreateWorkflowTemplate_Handler(srv interface{}, ct return interceptor(ctx, in, info, handler) } -func _WorkflowTemplateService_UpdateWorkflowTemplateVersion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(UpdateWorkflowTemplateVersionRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(WorkflowTemplateServiceServer).UpdateWorkflowTemplateVersion(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/api.WorkflowTemplateService/UpdateWorkflowTemplateVersion", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(WorkflowTemplateServiceServer).UpdateWorkflowTemplateVersion(ctx, req.(*UpdateWorkflowTemplateVersionRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _WorkflowTemplateService_CreateWorkflowTemplateVersion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CreateWorkflowTemplateRequest) if err := dec(in); err != nil { @@ -1752,10 +1703,6 @@ var _WorkflowTemplateService_serviceDesc = grpc.ServiceDesc{ MethodName: "CreateWorkflowTemplate", Handler: _WorkflowTemplateService_CreateWorkflowTemplate_Handler, }, - { - MethodName: "UpdateWorkflowTemplateVersion", - Handler: _WorkflowTemplateService_UpdateWorkflowTemplateVersion_Handler, - }, { MethodName: "CreateWorkflowTemplateVersion", Handler: _WorkflowTemplateService_CreateWorkflowTemplateVersion_Handler, diff --git a/api/workflow_template.pb.gw.go b/api/workflow_template.pb.gw.go index ffbadcc..78e735c 100644 --- a/api/workflow_template.pb.gw.go +++ b/api/workflow_template.pb.gw.go @@ -101,120 +101,6 @@ func local_request_WorkflowTemplateService_CreateWorkflowTemplate_0(ctx context. } -func request_WorkflowTemplateService_UpdateWorkflowTemplateVersion_0(ctx context.Context, marshaler runtime.Marshaler, client WorkflowTemplateServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq UpdateWorkflowTemplateVersionRequest - var metadata runtime.ServerMetadata - - newReader, berr := utilities.IOReaderFactory(req.Body) - if berr != nil { - return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr) - } - if err := marshaler.NewDecoder(newReader()).Decode(&protoReq.WorkflowTemplate); err != nil && err != io.EOF { - return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) - } - - var ( - val string - ok bool - err error - _ = err - ) - - val, ok = pathParams["namespace"] - if !ok { - return nil, metadata, status.Errorf(codes.InvalidArgument, "missing parameter %s", "namespace") - } - - protoReq.Namespace, err = runtime.String(val) - - if err != nil { - return nil, metadata, status.Errorf(codes.InvalidArgument, "type mismatch, parameter: %s, error: %v", "namespace", err) - } - - val, ok = pathParams["workflowTemplate.uid"] - if !ok { - return nil, metadata, status.Errorf(codes.InvalidArgument, "missing parameter %s", "workflowTemplate.uid") - } - - err = runtime.PopulateFieldFromPath(&protoReq, "workflowTemplate.uid", val) - - if err != nil { - return nil, metadata, status.Errorf(codes.InvalidArgument, "type mismatch, parameter: %s, error: %v", "workflowTemplate.uid", err) - } - - val, ok = pathParams["workflowTemplate.version"] - if !ok { - return nil, metadata, status.Errorf(codes.InvalidArgument, "missing parameter %s", "workflowTemplate.version") - } - - err = runtime.PopulateFieldFromPath(&protoReq, "workflowTemplate.version", val) - - if err != nil { - return nil, metadata, status.Errorf(codes.InvalidArgument, "type mismatch, parameter: %s, error: %v", "workflowTemplate.version", err) - } - - msg, err := client.UpdateWorkflowTemplateVersion(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) - return msg, metadata, err - -} - -func local_request_WorkflowTemplateService_UpdateWorkflowTemplateVersion_0(ctx context.Context, marshaler runtime.Marshaler, server WorkflowTemplateServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { - var protoReq UpdateWorkflowTemplateVersionRequest - var metadata runtime.ServerMetadata - - newReader, berr := utilities.IOReaderFactory(req.Body) - if berr != nil { - return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", berr) - } - if err := marshaler.NewDecoder(newReader()).Decode(&protoReq.WorkflowTemplate); err != nil && err != io.EOF { - return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) - } - - var ( - val string - ok bool - err error - _ = err - ) - - val, ok = pathParams["namespace"] - if !ok { - return nil, metadata, status.Errorf(codes.InvalidArgument, "missing parameter %s", "namespace") - } - - protoReq.Namespace, err = runtime.String(val) - - if err != nil { - return nil, metadata, status.Errorf(codes.InvalidArgument, "type mismatch, parameter: %s, error: %v", "namespace", err) - } - - val, ok = pathParams["workflowTemplate.uid"] - if !ok { - return nil, metadata, status.Errorf(codes.InvalidArgument, "missing parameter %s", "workflowTemplate.uid") - } - - err = runtime.PopulateFieldFromPath(&protoReq, "workflowTemplate.uid", val) - - if err != nil { - return nil, metadata, status.Errorf(codes.InvalidArgument, "type mismatch, parameter: %s, error: %v", "workflowTemplate.uid", err) - } - - val, ok = pathParams["workflowTemplate.version"] - if !ok { - return nil, metadata, status.Errorf(codes.InvalidArgument, "missing parameter %s", "workflowTemplate.version") - } - - err = runtime.PopulateFieldFromPath(&protoReq, "workflowTemplate.version", val) - - if err != nil { - return nil, metadata, status.Errorf(codes.InvalidArgument, "type mismatch, parameter: %s, error: %v", "workflowTemplate.version", err) - } - - msg, err := server.UpdateWorkflowTemplateVersion(ctx, &protoReq) - return msg, metadata, err - -} - func request_WorkflowTemplateService_CreateWorkflowTemplateVersion_0(ctx context.Context, marshaler runtime.Marshaler, client WorkflowTemplateServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var protoReq CreateWorkflowTemplateRequest var metadata runtime.ServerMetadata @@ -975,26 +861,6 @@ func RegisterWorkflowTemplateServiceHandlerServer(ctx context.Context, mux *runt }) - mux.Handle("PUT", pattern_WorkflowTemplateService_UpdateWorkflowTemplateVersion_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { - ctx, cancel := context.WithCancel(req.Context()) - defer cancel() - inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) - rctx, err := runtime.AnnotateIncomingContext(ctx, mux, req) - if err != nil { - runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) - return - } - resp, md, err := local_request_WorkflowTemplateService_UpdateWorkflowTemplateVersion_0(rctx, inboundMarshaler, server, req, pathParams) - ctx = runtime.NewServerMetadataContext(ctx, md) - if err != nil { - runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) - return - } - - forward_WorkflowTemplateService_UpdateWorkflowTemplateVersion_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) - - }) - mux.Handle("POST", pattern_WorkflowTemplateService_CreateWorkflowTemplateVersion_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() @@ -1216,26 +1082,6 @@ func RegisterWorkflowTemplateServiceHandlerClient(ctx context.Context, mux *runt }) - mux.Handle("PUT", pattern_WorkflowTemplateService_UpdateWorkflowTemplateVersion_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { - ctx, cancel := context.WithCancel(req.Context()) - defer cancel() - inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) - rctx, err := runtime.AnnotateContext(ctx, mux, req) - if err != nil { - runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) - return - } - resp, md, err := request_WorkflowTemplateService_UpdateWorkflowTemplateVersion_0(rctx, inboundMarshaler, client, req, pathParams) - ctx = runtime.NewServerMetadataContext(ctx, md) - if err != nil { - runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) - return - } - - forward_WorkflowTemplateService_UpdateWorkflowTemplateVersion_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) - - }) - mux.Handle("POST", pattern_WorkflowTemplateService_CreateWorkflowTemplateVersion_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() @@ -1402,8 +1248,6 @@ func RegisterWorkflowTemplateServiceHandlerClient(ctx context.Context, mux *runt var ( pattern_WorkflowTemplateService_CreateWorkflowTemplate_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 1, 5, 2, 2, 3}, []string{"apis", "v1beta1", "namespace", "workflow_templates"}, "", runtime.AssumeColonVerbOpt(true))) - pattern_WorkflowTemplateService_UpdateWorkflowTemplateVersion_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 1, 5, 2, 2, 3, 1, 0, 4, 1, 5, 4, 2, 5, 1, 0, 4, 1, 5, 6}, []string{"apis", "v1beta1", "namespace", "workflow_templates", "workflowTemplate.uid", "versions", "workflowTemplate.version"}, "", runtime.AssumeColonVerbOpt(true))) - pattern_WorkflowTemplateService_CreateWorkflowTemplateVersion_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 1, 5, 2, 2, 3, 1, 0, 4, 1, 5, 4, 2, 5}, []string{"apis", "v1beta1", "namespace", "workflow_templates", "workflowTemplate.uid", "versions"}, "", runtime.AssumeColonVerbOpt(true))) pattern_WorkflowTemplateService_GetWorkflowTemplate_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 1, 5, 2, 2, 3, 1, 0, 4, 1, 5, 4}, []string{"apis", "v1beta1", "namespace", "workflow_templates", "uid"}, "", runtime.AssumeColonVerbOpt(true))) @@ -1424,8 +1268,6 @@ var ( var ( forward_WorkflowTemplateService_CreateWorkflowTemplate_0 = runtime.ForwardResponseMessage - forward_WorkflowTemplateService_UpdateWorkflowTemplateVersion_0 = runtime.ForwardResponseMessage - forward_WorkflowTemplateService_CreateWorkflowTemplateVersion_0 = runtime.ForwardResponseMessage forward_WorkflowTemplateService_GetWorkflowTemplate_0 = runtime.ForwardResponseMessage diff --git a/api/workflow_template.proto b/api/workflow_template.proto index 367c6df..75a7a6d 100644 --- a/api/workflow_template.proto +++ b/api/workflow_template.proto @@ -13,13 +13,6 @@ service WorkflowTemplateService { }; } - rpc UpdateWorkflowTemplateVersion (UpdateWorkflowTemplateVersionRequest) returns (WorkflowTemplate) { - option (google.api.http) = { - put: "/apis/v1beta1/{namespace}/workflow_templates/{workflowTemplate.uid}/versions/{workflowTemplate.version}" - body: "workflowTemplate" - }; - } - rpc CreateWorkflowTemplateVersion (CreateWorkflowTemplateRequest) returns (WorkflowTemplate) { option (google.api.http) = { post: "/apis/v1beta1/{namespace}/workflow_templates/{workflowTemplate.uid}/versions" diff --git a/cmd/goose/goose.go b/cmd/goose/goose.go index dff0f80..b701d11 100644 --- a/cmd/goose/goose.go +++ b/cmd/goose/goose.go @@ -4,7 +4,6 @@ package main import ( "flag" - "fmt" "github.com/jmoiron/sqlx" _ "github.com/onepanelio/core/db" v1 "github.com/onepanelio/core/pkg" @@ -38,10 +37,8 @@ func main() { log.Fatalf("Failed to get system config: %v", err) } - databaseDataSourceName := fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=disable", - config["databaseHost"], config["databaseUsername"], config["databasePassword"], config["databaseName"]) - - db := sqlx.MustConnect(config["databaseDriverName"], databaseDataSourceName) + dbDriverName, dbDataSourceName := config.DatabaseConnection() + db := sqlx.MustConnect(dbDriverName, dbDataSourceName) command := args[0] diff --git a/db/20200704151301_update_cvat_workspace_template.go b/db/20200704151301_update_cvat_workspace_template.go new file mode 100644 index 0000000..731b8ac --- /dev/null +++ b/db/20200704151301_update_cvat_workspace_template.go @@ -0,0 +1,167 @@ +package migration + +import ( + "database/sql" + v1 "github.com/onepanelio/core/pkg" + uid2 "github.com/onepanelio/core/pkg/util/uid" + "github.com/pressly/goose" + "time" +) + +const cvatWorkspaceTemplate3 = `# Docker containers that are part of the Workspace +containers: +- name: cvat-db + image: postgres:10-alpine + env: + - name: POSTGRES_USER + value: root + - name: POSTGRES_DB + value: cvat + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/psql/data + ports: + - containerPort: 5432 + name: tcp + volumeMounts: + - name: db + mountPath: /var/lib/psql +- name: cvat-redis + image: redis:4.0-alpine + ports: + - containerPort: 6379 + name: tcp +- name: cvat + image: onepanel/cvat:v0.7.10-stable + env: + - name: DJANGO_MODWSGI_EXTRA_ARGS + value: "" + - name: ALLOWED_HOSTS + value: '*' + - name: CVAT_REDIS_HOST + value: localhost + - name: CVAT_POSTGRES_HOST + value: localhost + - name: CVAT_SHARE_URL + value: /home/django/data + ports: + - containerPort: 8080 + name: http + volumeMounts: + - name: data + mountPath: /home/django/data + - name: keys + mountPath: /home/django/keys + - name: logs + mountPath: /home/django/logs + - name: models + mountPath: /home/django/models + - name: share + mountPath: /home/django/share +- name: cvat-ui + image: onepanel/cvat-ui:v0.7.10-stable + ports: + - containerPort: 80 + name: http +# Uncomment following lines to enable S3 FileSyncer +# Refer to https://docs.onepanel.ai/docs/getting-started/use-cases/computervision/annotation/cvat/cvat_quick_guide#setting-up-environment-variables +#- name: filesyncer +# image: onepanel/filesyncer:v0.0.4 +# command: ['python3', 'main.py'] +# volumeMounts: +# - name: share +# mountPath: /mnt/share +ports: +- name: cvat-ui + port: 80 + protocol: TCP + targetPort: 80 +- name: cvat + port: 8080 + protocol: TCP + targetPort: 8080 +routes: +- match: + - uri: + regex: /api/.*|/git/.*|/tensorflow/.*|/auto_annotation/.*|/analytics/.*|/static/.*|/admin/.*|/documentation/.*|/dextr/.*|/reid/.* + - queryParams: + id: + regex: \d+.* + route: + - destination: + port: + number: 8080 + timeout: 600s +- match: + - uri: + prefix: / + route: + - destination: + port: + number: 80 + timeout: 600s +# DAG Workflow to be executed once a Workspace action completes (optional) +# Uncomment the lines below if you want to send Slack notifications +#postExecutionWorkflow: +# entrypoint: main +# templates: +# - name: main +# dag: +# tasks: +# - name: slack-notify +# template: slack-notify +# - name: slack-notify +# container: +# image: technosophos/slack-notify +# args: +# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify +# command: +# - sh +# - -c +` + +func init() { + goose.AddMigration(Up20200704151301, Down20200704151301) +} + +// Up20200704151301 updates the CVAT template to a new version. +func Up20200704151301(tx *sql.Tx) error { + // This code is executed when the migration is applied. + + time.Sleep(2 * time.Second) + + client, err := getClient() + if err != nil { + return err + } + + namespaces, err := client.ListOnepanelEnabledNamespaces() + if err != nil { + return err + } + + uid, err := uid2.GenerateUID(cvatTemplateName, 30) + if err != nil { + return err + } + workspaceTemplate := &v1.WorkspaceTemplate{ + UID: uid, + Name: cvatTemplateName, + Manifest: cvatWorkspaceTemplate3, + } + + for _, namespace := range namespaces { + if _, err := client.UpdateWorkspaceTemplate(namespace.Name, workspaceTemplate); err != nil { + return err + } + } + + return nil +} + +// Down20200704151301 removes the CVAT template update +func Down20200704151301(tx *sql.Tx) error { + // This code is executed when the migration is rolled back. + return nil +} diff --git a/db/db.go b/db/db.go deleted file mode 100644 index 7df09c7..0000000 --- a/db/db.go +++ /dev/null @@ -1,25 +0,0 @@ -package migration - -import ( - "fmt" - "github.com/jmoiron/sqlx" - v1 "github.com/onepanelio/core/pkg" -) - -func getClient() (*v1.Client, error) { - kubeConfig := v1.NewConfig() - client, err := v1.NewClient(kubeConfig, nil, nil) - if err != nil { - return nil, err - } - config, err := client.GetSystemConfig() - if err != nil { - return nil, err - } - - databaseDataSourceName := fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=disable", - config["databaseHost"], config["databaseUsername"], config["databasePassword"], config["databaseName"]) - client.DB = v1.NewDB(sqlx.MustConnect(config["databaseDriverName"], databaseDataSourceName)) - - return client, nil -} diff --git a/db/20200525160514_add_jupyter_workspace_template.go b/db/go/20200525160514_add_jupyter_workspace_template.go similarity index 94% rename from db/20200525160514_add_jupyter_workspace_template.go rename to db/go/20200525160514_add_jupyter_workspace_template.go index 3f12262..fe922b2 100644 --- a/db/20200525160514_add_jupyter_workspace_template.go +++ b/db/go/20200525160514_add_jupyter_workspace_template.go @@ -71,7 +71,7 @@ routes: const jupyterLabTemplateName = "JupyterLab" -func init() { +func initialize20200525160514() { goose.AddMigration(Up20200525160514, Down20200525160514) } @@ -81,6 +81,15 @@ func Up20200525160514(tx *sql.Tx) error { return err } + migrationsRan, err := getRanSQLMigrations(client) + if err != nil { + return err + } + + if _, ok := migrationsRan[20200525160514]; ok { + return nil + } + namespaces, err := client.ListOnepanelEnabledNamespaces() if err != nil { return err diff --git a/db/20200528140124_add_cvat_workspace_template.go b/db/go/20200528140124_add_cvat_workspace_template.go similarity index 95% rename from db/20200528140124_add_cvat_workspace_template.go rename to db/go/20200528140124_add_cvat_workspace_template.go index 06fda51..b705007 100644 --- a/db/20200528140124_add_cvat_workspace_template.go +++ b/db/go/20200528140124_add_cvat_workspace_template.go @@ -110,7 +110,7 @@ routes: const cvatTemplateName = "CVAT" -func init() { +func initialize20200528140124() { goose.AddMigration(Up20200528140124, Down20200528140124) } @@ -125,6 +125,15 @@ func Up20200528140124(tx *sql.Tx) error { return err } + migrationsRan, err := getRanSQLMigrations(client) + if err != nil { + return err + } + + if _, ok := migrationsRan[20200528140124]; ok { + return nil + } + namespaces, err := client.ListOnepanelEnabledNamespaces() if err != nil { return err diff --git a/db/20200605090509_add_pytorch_workflow_template.go b/db/go/20200605090509_add_pytorch_workflow_template.go similarity index 95% rename from db/20200605090509_add_pytorch_workflow_template.go rename to db/go/20200605090509_add_pytorch_workflow_template.go index 4cd8fe2..7c12b70 100644 --- a/db/20200605090509_add_pytorch_workflow_template.go +++ b/db/go/20200605090509_add_pytorch_workflow_template.go @@ -88,7 +88,7 @@ templates: const pytorchMnistWorkflowTemplateName = "PyTorch Training" -func init() { +func initialize20200605090509() { goose.AddMigration(Up20200605090509, Down20200605090509) } @@ -101,6 +101,15 @@ func Up20200605090509(tx *sql.Tx) error { return err } + migrationsRan, err := getRanSQLMigrations(client) + if err != nil { + return err + } + + if _, ok := migrationsRan[20200605090509]; ok { + return nil + } + namespaces, err := client.ListOnepanelEnabledNamespaces() if err != nil { return err diff --git a/db/20200605090535_add_tensorflow_workflow_template.go b/db/go/20200605090535_add_tensorflow_workflow_template.go similarity index 95% rename from db/20200605090535_add_tensorflow_workflow_template.go rename to db/go/20200605090535_add_tensorflow_workflow_template.go index a543788..6f7b224 100644 --- a/db/20200605090535_add_tensorflow_workflow_template.go +++ b/db/go/20200605090535_add_tensorflow_workflow_template.go @@ -88,7 +88,7 @@ templates: const tensorflowWorkflowTemplateName = "TensorFlow Training" -func init() { +func initialize20200605090535() { goose.AddMigration(Up20200605090535, Down20200605090535) } @@ -101,6 +101,15 @@ func Up20200605090535(tx *sql.Tx) error { return err } + migrationsRan, err := getRanSQLMigrations(client) + if err != nil { + return err + } + + if _, ok := migrationsRan[20200605090535]; ok { + return nil + } + namespaces, err := client.ListOnepanelEnabledNamespaces() if err != nil { return err diff --git a/db/20200626113635_update_cvat_workspace_template.go b/db/go/20200626113635_update_cvat_workspace_template.go similarity index 95% rename from db/20200626113635_update_cvat_workspace_template.go rename to db/go/20200626113635_update_cvat_workspace_template.go index 451ab4a..55584f1 100644 --- a/db/20200626113635_update_cvat_workspace_template.go +++ b/db/go/20200626113635_update_cvat_workspace_template.go @@ -5,7 +5,6 @@ import ( v1 "github.com/onepanelio/core/pkg" uid2 "github.com/onepanelio/core/pkg/util/uid" "github.com/pressly/goose" - "time" ) const cvatWorkspaceTemplate2 = `# Docker containers that are part of the Workspace @@ -119,21 +118,27 @@ routes: # - -c ` -func init() { +func initialize20200626113635() { goose.AddMigration(Up20200626113635, Down20200626113635) } // Up20200626113635 updates the CVAT template to a new version. func Up20200626113635(tx *sql.Tx) error { // This code is executed when the migration is applied. - - time.Sleep(2 * time.Second) - client, err := getClient() if err != nil { return err } + migrationsRan, err := getRanSQLMigrations(client) + if err != nil { + return err + } + + if _, ok := migrationsRan[20200626113635]; ok { + return nil + } + namespaces, err := client.ListOnepanelEnabledNamespaces() if err != nil { return err diff --git a/db/go/db.go b/db/go/db.go new file mode 100644 index 0000000..a4fb5c4 --- /dev/null +++ b/db/go/db.go @@ -0,0 +1,53 @@ +package migration + +import ( + sq "github.com/Masterminds/squirrel" + "github.com/jmoiron/sqlx" + v1 "github.com/onepanelio/core/pkg" +) + +// Initialize sets up the go migrations. +func Initialize() { + initialize20200525160514() + initialize20200528140124() + initialize20200605090509() + initialize20200605090535() + initialize20200626113635() +} + +func getClient() (*v1.Client, error) { + kubeConfig := v1.NewConfig() + client, err := v1.NewClient(kubeConfig, nil, nil) + if err != nil { + return nil, err + } + config, err := client.GetSystemConfig() + if err != nil { + return nil, err + } + + dbDriverName, dbDataSourceName := config.DatabaseConnection() + client.DB = v1.NewDB(sqlx.MustConnect(dbDriverName, dbDataSourceName)) + + return client, nil +} + +// getRanSQLMigrations returns a map where each key is a sql migration version ran. +func getRanSQLMigrations(client *v1.Client) (map[uint64]bool, error) { + sb := sq.StatementBuilder.PlaceholderFormat(sq.Dollar) + + query := sb.Select("version_id"). + From("goose_db_version") + + versions := make([]uint64, 0) + if err := client.Selectx(&versions, query); err != nil { + return nil, err + } + + result := make(map[uint64]bool) + for _, version := range versions { + result[version] = true + } + + return result, nil +} diff --git a/db/20191210141652_create_workflow_templates.sql b/db/sql/20191210141652_create_workflow_templates.sql similarity index 100% rename from db/20191210141652_create_workflow_templates.sql rename to db/sql/20191210141652_create_workflow_templates.sql diff --git a/db/20191210181732_create_workflow_template_versions.sql b/db/sql/20191210181732_create_workflow_template_versions.sql similarity index 100% rename from db/20191210181732_create_workflow_template_versions.sql rename to db/sql/20191210181732_create_workflow_template_versions.sql diff --git a/db/20200123130105_add_latest_workflow_template_version.sql b/db/sql/20200123130105_add_latest_workflow_template_version.sql similarity index 100% rename from db/20200123130105_add_latest_workflow_template_version.sql rename to db/sql/20200123130105_add_latest_workflow_template_version.sql diff --git a/db/20200204103708_add_is_archived_to_workflow_templates.sql b/db/sql/20200204103708_add_is_archived_to_workflow_templates.sql similarity index 100% rename from db/20200204103708_add_is_archived_to_workflow_templates.sql rename to db/sql/20200204103708_add_is_archived_to_workflow_templates.sql diff --git a/db/20200407113554_remove_workflow_template_versions_table.sql b/db/sql/20200407113554_remove_workflow_template_versions_table.sql similarity index 100% rename from db/20200407113554_remove_workflow_template_versions_table.sql rename to db/sql/20200407113554_remove_workflow_template_versions_table.sql diff --git a/db/20200413141037_create_workflow_execution.sql b/db/sql/20200413141037_create_workflow_execution.sql similarity index 100% rename from db/20200413141037_create_workflow_execution.sql rename to db/sql/20200413141037_create_workflow_execution.sql diff --git a/db/20200420112057_cache_workflow_template_versions_count.sql b/db/sql/20200420112057_cache_workflow_template_versions_count.sql similarity index 100% rename from db/20200420112057_cache_workflow_template_versions_count.sql rename to db/sql/20200420112057_cache_workflow_template_versions_count.sql diff --git a/db/20200421103946_add_back_workflow_template_versions.sql b/db/sql/20200421103946_add_back_workflow_template_versions.sql similarity index 100% rename from db/20200421103946_add_back_workflow_template_versions.sql rename to db/sql/20200421103946_add_back_workflow_template_versions.sql diff --git a/db/20200422140125_add_cron_workflows.sql b/db/sql/20200422140125_add_cron_workflows.sql similarity index 100% rename from db/20200422140125_add_cron_workflows.sql rename to db/sql/20200422140125_add_cron_workflows.sql diff --git a/db/20200422160902_add_labels.sql b/db/sql/20200422160902_add_labels.sql similarity index 100% rename from db/20200422160902_add_labels.sql rename to db/sql/20200422160902_add_labels.sql diff --git a/db/20200424132932_add_started_and_version_to_workflow_executions.sql b/db/sql/20200424132932_add_started_and_version_to_workflow_executions.sql similarity index 100% rename from db/20200424132932_add_started_and_version_to_workflow_executions.sql rename to db/sql/20200424132932_add_started_and_version_to_workflow_executions.sql diff --git a/db/20200425172611_workspace_templates.sql b/db/sql/20200425172611_workspace_templates.sql similarity index 100% rename from db/20200425172611_workspace_templates.sql rename to db/sql/20200425172611_workspace_templates.sql diff --git a/db/20200425173049_workspace_template_versions.sql b/db/sql/20200425173049_workspace_template_versions.sql similarity index 100% rename from db/20200425173049_workspace_template_versions.sql rename to db/sql/20200425173049_workspace_template_versions.sql diff --git a/db/20200429141030_add_manifest_to_crons.sql b/db/sql/20200429141030_add_manifest_to_crons.sql similarity index 100% rename from db/20200429141030_add_manifest_to_crons.sql rename to db/sql/20200429141030_add_manifest_to_crons.sql diff --git a/db/20200502184729_workspaces.sql b/db/sql/20200502184729_workspaces.sql similarity index 100% rename from db/20200502184729_workspaces.sql rename to db/sql/20200502184729_workspaces.sql diff --git a/db/20200504161141_add_parameters_to_workflow_executions.sql b/db/sql/20200504161141_add_parameters_to_workflow_executions.sql similarity index 100% rename from db/20200504161141_add_parameters_to_workflow_executions.sql rename to db/sql/20200504161141_add_parameters_to_workflow_executions.sql diff --git a/db/20200510202248_add_is_system_to_workflow_templates.sql b/db/sql/20200510202248_add_is_system_to_workflow_templates.sql similarity index 100% rename from db/20200510202248_add_is_system_to_workflow_templates.sql rename to db/sql/20200510202248_add_is_system_to_workflow_templates.sql diff --git a/db/20200511164307_add_path_to_workspaces.sql b/db/sql/20200511164307_add_path_to_workspaces.sql similarity index 100% rename from db/20200511164307_add_path_to_workspaces.sql rename to db/sql/20200511164307_add_path_to_workspaces.sql diff --git a/db/20200512182928_add_updated_at_to_workspaces.sql b/db/sql/20200512182928_add_updated_at_to_workspaces.sql similarity index 100% rename from db/20200512182928_add_updated_at_to_workspaces.sql rename to db/sql/20200512182928_add_updated_at_to_workspaces.sql diff --git a/db/20200518163745_add_conditional_unique_index_to_workflow_templates.sql b/db/sql/20200518163745_add_conditional_unique_index_to_workflow_templates.sql similarity index 100% rename from db/20200518163745_add_conditional_unique_index_to_workflow_templates.sql rename to db/sql/20200518163745_add_conditional_unique_index_to_workflow_templates.sql diff --git a/db/20200521112757_add_namespace_cron_workflows.sql b/db/sql/20200521112757_add_namespace_cron_workflows.sql similarity index 100% rename from db/20200521112757_add_namespace_cron_workflows.sql rename to db/sql/20200521112757_add_namespace_cron_workflows.sql diff --git a/db/20200521124859_add_unique_index_cron_workflows_uid_namespace.sql b/db/sql/20200521124859_add_unique_index_cron_workflows_uid_namespace.sql similarity index 100% rename from db/20200521124859_add_unique_index_cron_workflows_uid_namespace.sql rename to db/sql/20200521124859_add_unique_index_cron_workflows_uid_namespace.sql diff --git a/db/20200521145827_add_is_archived_to_cron_wfs_and_wf_execs.sql b/db/sql/20200521145827_add_is_archived_to_cron_wfs_and_wf_execs.sql similarity index 100% rename from db/20200521145827_add_is_archived_to_cron_wfs_and_wf_execs.sql rename to db/sql/20200521145827_add_is_archived_to_cron_wfs_and_wf_execs.sql diff --git a/db/20200522164526_alter_workflow_executions_column_sizes.sql b/db/sql/20200522164526_alter_workflow_executions_column_sizes.sql similarity index 100% rename from db/20200522164526_alter_workflow_executions_column_sizes.sql rename to db/sql/20200522164526_alter_workflow_executions_column_sizes.sql diff --git a/db/20200522171359_alter_cron_workflows_column_sizes.sql b/db/sql/20200522171359_alter_cron_workflows_column_sizes.sql similarity index 100% rename from db/20200522171359_alter_cron_workflows_column_sizes.sql rename to db/sql/20200522171359_alter_cron_workflows_column_sizes.sql diff --git a/db/20200524114047_update_is_archived_to_not_null.sql b/db/sql/20200524114047_update_is_archived_to_not_null.sql similarity index 100% rename from db/20200524114047_update_is_archived_to_not_null.sql rename to db/sql/20200524114047_update_is_archived_to_not_null.sql diff --git a/db/20200616194847_remove_workflow_template_versions_uid.sql b/db/sql/20200616194847_remove_workflow_template_versions_uid.sql similarity index 68% rename from db/20200616194847_remove_workflow_template_versions_uid.sql rename to db/sql/20200616194847_remove_workflow_template_versions_uid.sql index b66607e..708261d 100644 --- a/db/20200616194847_remove_workflow_template_versions_uid.sql +++ b/db/sql/20200616194847_remove_workflow_template_versions_uid.sql @@ -2,5 +2,4 @@ ALTER TABLE workflow_template_versions DROP COLUMN uid; -- +goose Down -ALTER TABLE workflow_template_versions ADD COLUMN uid VARCHAR(30); UPDATE workflow_template_versions SET uid = version::text; \ No newline at end of file diff --git a/db/20200617122509_remove_url_from_workspaces.sql b/db/sql/20200617122509_remove_url_from_workspaces.sql similarity index 100% rename from db/20200617122509_remove_url_from_workspaces.sql rename to db/sql/20200617122509_remove_url_from_workspaces.sql diff --git a/db/sql/20200630112657_update_version_types.sql b/db/sql/20200630112657_update_version_types.sql new file mode 100644 index 0000000..5ce6627 --- /dev/null +++ b/db/sql/20200630112657_update_version_types.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- SQL in this section is executed when the migration is applied. +ALTER TABLE workflow_template_versions ALTER COLUMN version TYPE BIGINT; +ALTER TABLE workspace_template_versions ALTER COLUMN version TYPE BIGINT; +ALTER TABLE workspaces ALTER COLUMN workspace_template_version TYPE BIGINT; + +-- +goose Down +-- SQL in this section is executed when the migration is rolled back. +ALTER TABLE workflow_template_versions ALTER COLUMN version TYPE INT; +ALTER TABLE workspace_template_versions ALTER COLUMN version TYPE INT; +ALTER TABLE workspaces ALTER COLUMN workspace_template_version TYPE INT; \ No newline at end of file diff --git a/main.go b/main.go index e10227b..c349cd4 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( "context" "flag" "fmt" - _ "github.com/onepanelio/core/db" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" @@ -23,6 +22,7 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/runtime" "github.com/jmoiron/sqlx" "github.com/onepanelio/core/api" + migrations "github.com/onepanelio/core/db/go" v1 "github.com/onepanelio/core/pkg" "github.com/onepanelio/core/pkg/util/env" "github.com/onepanelio/core/server" @@ -67,16 +67,20 @@ func main() { log.Fatalf("Failed to get system config: %v", err) } - databaseDataSourceName := fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=disable", - sysConfig["databaseHost"], sysConfig["databaseUsername"], sysConfig["databasePassword"], sysConfig["databaseName"]) - + dbDriverName, databaseDataSourceName := sysConfig.DatabaseConnection() // sqlx.MustConnect will panic when it can't connect to DB. In that case, this whole application will crash. // This is okay, as the pod will restart and try connecting to DB again. // dbDriverName may be nil, but sqlx will then panic. - dbDriverName := sysConfig.DatabaseDriverName() - db := sqlx.MustConnect(*dbDriverName, databaseDataSourceName) - if err := goose.Run("up", db.DB, "db"); err != nil { - log.Fatalf("Failed to run database migrations: %v", err) + db := sqlx.MustConnect(dbDriverName, databaseDataSourceName) + goose.SetTableName("goose_db_version") + if err := goose.Run("up", db.DB, "db/sql"); err != nil { + log.Fatalf("Failed to run database sql migrations: %v", err) + } + + goose.SetTableName("goose_db_go_version") + migrations.Initialize() + if err := goose.Run("up", db.DB, "db/go"); err != nil { + log.Fatalf("Failed to run database go migrations: %v", err) } s := startRPCServer(v1.NewDB(db), kubeConfig, sysConfig, stopCh) diff --git a/pkg/client_test.go b/pkg/client_test.go index d5e1c85..4a0d4d3 100644 --- a/pkg/client_test.go +++ b/pkg/client_test.go @@ -1,10 +1,18 @@ package v1 import ( + "flag" + "fmt" + argoFake "github.com/argoproj/argo/pkg/client/clientset/versioned/fake" + "github.com/jmoiron/sqlx" + "github.com/pressly/goose" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" + "log" + "os" + "testing" ) var ( @@ -15,6 +23,20 @@ var ( }, } + configArtifactRepository = `archiveLogs: true +s3: + keyFormat: artifacts/{{workflow.namespace}}/{{workflow.name}}/{{pod.name}} + bucket: test.onepanel.io + endpoint: s3.amazonaws.com + insecure: false + region: us-west-2 + accessKeySecret: + name: onepanel + key: artifactRepositoryS3AccessKey + secretKeySecret: + name: onepanel + key: artifactRepositoryS3SecretKey` + mockSystemConfigMap = &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "onepanel", @@ -22,6 +44,8 @@ var ( }, Data: map[string]string{ "ONEPANEL_HOST": "demo.onepanel.site", + "ONEPANEL_DOMAIN": "demo.onepanel.site", + "artifactRepository": configArtifactRepository, "applicationNodePoolLabel": "beta.kubernetes.io/instance-type", "applicationNodePoolOptions": ` - name: 'CPU: 2, RAM: 8GB' @@ -34,8 +58,60 @@ var ( `, }, } + + database *sqlx.DB ) -func NewTestClient(objects ...runtime.Object) (client *Client) { - return &Client{Interface: fake.NewSimpleClientset(objects...)} +var flagDatabaseService = flag.String("db", "localhost", "Name to connect to db, defaults to localhost") + +func TestMain(m *testing.M) { + // call flag.Parse() here if TestMain uses flags + flag.Parse() + + databaseDataSourceName := fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=disable", + *flagDatabaseService, "admin", "tester", "onepanel") + + dbDriverName := "postgres" + database = sqlx.MustConnect(dbDriverName, databaseDataSourceName) + + // We don't run the go migrations as those setup data that we don't use in our testing + if err := goose.Run("up", database.DB, "../db/sql"); err != nil { + log.Fatalf("Failed to run database migrations: %v", err) + } + + os.Exit(m.Run()) +} + +func NewTestClient(db *sqlx.DB, objects ...runtime.Object) (client *Client) { + k8sFake := fake.NewSimpleClientset(objects...) + argoFakeClient := argoFake.NewSimpleClientset() + + return &Client{ + Interface: k8sFake, + DB: NewDB(db), + argoprojV1alpha1: argoFakeClient.ArgoprojV1alpha1(), + } +} + +func DefaultTestClient() *Client { + return NewTestClient(database, mockSystemConfigMap, mockSystemSecret) +} + +func clearDatabase(t *testing.T) { + // We do not delete from goose_db_version as we need it to mark the migrations as ran. + query := ` + DELETE FROM labels; + DELETE FROM workspaces; + DELETE FROM workflow_executions; + DELETE FROM cron_workflows; + DELETE FROM workspace_templates; + DELETE FROM workflow_templates; + DELETE FROM workspace_template_versions; + DELETE FROM workflow_template_versions; + ` + + _, err := database.Exec(query) + if err != nil { + t.Fatal(err) + } } diff --git a/pkg/config.go b/pkg/config.go index 5f75c9a..0da9934 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -36,20 +36,19 @@ func (c *Client) GetSystemConfig() (config SystemConfig, err error) { } namespace := "onepanel" - configMap, err := c.getConfigMap(namespace, "onepanel") - if err != nil { - return - } - config = configMap.Data + name := "onepanel" - secret, err := c.GetSecret(namespace, "onepanel") + configMap, err := c.getConfigMap(namespace, name) if err != nil { return } - databaseUsername, _ := base64.StdEncoding.DecodeString(secret.Data["databaseUsername"]) - config["databaseUsername"] = string(databaseUsername) - databasePassword, _ := base64.StdEncoding.DecodeString(secret.Data["databasePassword"]) - config["databasePassword"] = string(databasePassword) + + secret, err := c.GetSecret(namespace, name) + if err != nil { + return + } + + config, err = NewSystemConfig(configMap, secret) c.systemConfig = config diff --git a/pkg/namespace_test.go b/pkg/namespace_test.go index 2d773d9..f0985f9 100644 --- a/pkg/namespace_test.go +++ b/pkg/namespace_test.go @@ -21,8 +21,9 @@ func testCreateNamespace(c *Client) { }) } } -func TestListNamespace(t *testing.T) { - c := NewTestClient() + +func TestClient_ListNamespace(t *testing.T) { + c := DefaultTestClient() testCreateNamespace(c) diff --git a/pkg/secret_test.go b/pkg/secret_test.go index 850ea80..b83856c 100644 --- a/pkg/secret_test.go +++ b/pkg/secret_test.go @@ -6,8 +6,8 @@ import ( "github.com/stretchr/testify/assert" ) -func TestCreateSecret(t *testing.T) { - c := NewTestClient() +func TestClient_CreateSecret(t *testing.T) { + c := DefaultTestClient() err := c.CreateSecret("namespace", &Secret{ Name: "name", @@ -15,8 +15,8 @@ func TestCreateSecret(t *testing.T) { assert.Nil(t, err) } -func TestGetSecret(t *testing.T) { - c := NewTestClient() +func TestClient_GetSecret(t *testing.T) { + c := DefaultTestClient() err := c.CreateSecret("namespace", &Secret{ Name: "name", diff --git a/pkg/workflow_execution.go b/pkg/workflow_execution.go index d7c7b79..db067fb 100644 --- a/pkg/workflow_execution.go +++ b/pkg/workflow_execution.go @@ -2,6 +2,7 @@ package v1 import ( "bufio" + "database/sql" "encoding/json" "errors" "fmt" @@ -85,6 +86,18 @@ func UnmarshalWorkflows(wfBytes []byte, strict bool) (wfs []wfv1.Workflow, err e return } +// getWorkflowsFromWorkflowTemplate parses the WorkflowTemplate manifest and returns the argo workflows from it +func getWorkflowsFromWorkflowTemplate(wt *WorkflowTemplate) (wfs []wfv1.Workflow, err error) { + manifest, err := wt.GetWorkflowManifestBytes() + if err != nil { + return nil, err + } + + wfs, err = UnmarshalWorkflows(manifest, true) + + return +} + // appendArtifactRepositoryConfigIfMissing appends default artifact repository config to artifacts that have a key. // Artifacts that contain anything other than key are skipped. func injectArtifactRepositoryConfig(artifact *wfv1.Artifact, namespaceConfig *NamespaceConfig) { @@ -265,11 +278,19 @@ func (c *Client) injectAutomatedFields(namespace string, wf *wfv1.Workflow, opts return } +// ArchiveWorkflowExecution marks a WorkflowExecution as archived in database +// and deletes the argo workflow. +// +// If the database record does not exist, we still try to delete the argo workflow record. +// No errors are returned if the records do not exist. func (c *Client) ArchiveWorkflowExecution(namespace, uid string) error { - _, err := sb.Update("workflow_executions").Set("is_archived", true).Where(sq.Eq{ - "uid": uid, - "namespace": namespace, - }).RunWith(c.DB).Exec() + _, err := sb.Update("workflow_executions"). + Set("is_archived", true). + Where(sq.Eq{ + "uid": uid, + "namespace": namespace, + }).RunWith(c.DB). + Exec() if err != nil { return err } @@ -285,11 +306,10 @@ func (c *Client) ArchiveWorkflowExecution(namespace, uid string) error { return nil } -/* - Name is == to UID, no user friendly name. - Workflow execution name == uid, example: name = my-friendly-wf-name-8skjz, uid = my-friendly-wf-name-8skjz -*/ -func (c *Client) createWorkflow(namespace string, workflowTemplateId uint64, workflowTemplateVersionId uint64, wf *wfv1.Workflow, opts *WorkflowExecutionOptions) (newDbId uint64, createdWorkflow *wfv1.Workflow, err error) { +// createWorkflow creates the workflow in the database and argo. +// Name is == to UID, no user friendly name. +// Workflow execution name == uid, example: name = my-friendly-wf-name-8skjz, uid = my-friendly-wf-name-8skjz +func (c *Client) createWorkflow(namespace string, workflowTemplateID uint64, workflowTemplateVersionID uint64, wf *wfv1.Workflow, opts *WorkflowExecutionOptions) (createdWorkflow *WorkflowExecution, err error) { if opts == nil { opts = &WorkflowExecutionOptions{} } @@ -330,34 +350,41 @@ func (c *Client) createWorkflow(namespace string, workflowTemplateId uint64, wor wf.ObjectMeta.Labels = opts.Labels } - err = injectWorkflowExecutionStatusCaller(wf, wfv1.NodeRunning) - if err != nil { - return 0, nil, err + if err = injectWorkflowExecutionStatusCaller(wf, wfv1.NodeRunning); err != nil { + return nil, err } - err = injectExitHandlerWorkflowExecutionStatistic(wf, &workflowTemplateId) - if err != nil { - return 0, nil, err + if err = injectExitHandlerWorkflowExecutionStatistic(wf, &workflowTemplateID); err != nil { + return nil, err } if err = c.injectAutomatedFields(namespace, wf, opts); err != nil { - return 0, nil, err + return nil, err } - createdWorkflow, err = c.ArgoprojV1alpha1().Workflows(namespace).Create(wf) + createdArgoWorkflow, err := c.ArgoprojV1alpha1().Workflows(namespace).Create(wf) if err != nil { - return 0, nil, err + return nil, err } - uid, err := uid2.GenerateUID(createdWorkflow.Name, 63) - if err != nil { - return 0, nil, err + createdWorkflow = &WorkflowExecution{ + Name: createdArgoWorkflow.Name, + CreatedAt: createdArgoWorkflow.CreationTimestamp.UTC(), + ArgoWorkflow: createdArgoWorkflow, + WorkflowTemplate: &WorkflowTemplate{ + WorkflowTemplateVersionID: workflowTemplateVersionID, + }, + Parameters: opts.Parameters, } + + if err = createdWorkflow.GenerateUID(createdArgoWorkflow.Name); err != nil { + return nil, err + } + //Create an entry for workflow_executions statistic //CURL code will hit the API endpoint that will update the db row - newDbId, err = c.insertPreWorkflowExecutionStatistic(namespace, createdWorkflow.Name, workflowTemplateVersionId, createdWorkflow.CreationTimestamp.UTC(), uid, opts.Parameters) - if err != nil { - return 0, nil, err + if err := c.createWorkflowExecutionDB(namespace, createdWorkflow); err != nil { + return nil, err } return @@ -376,7 +403,9 @@ func (c *Client) ValidateWorkflowExecution(namespace string, manifest []byte) (e wftmplGetter := templateresolution.WrapWorkflowTemplateInterface(c.ArgoprojV1alpha1().WorkflowTemplates(namespace)) for _, wf := range workflows { - c.injectAutomatedFields(namespace, &wf, &WorkflowExecutionOptions{}) + if err = c.injectAutomatedFields(namespace, &wf, &WorkflowExecutionOptions{}); err != nil { + return err + } _, err = validate.ValidateWorkflow(wftmplGetter, &wf, validate.ValidateOpts{}) if err != nil { return @@ -398,16 +427,22 @@ func (c *Client) ValidateWorkflowExecution(namespace string, manifest []byte) (e } // CreateWorkflowExecution creates an argo workflow execution and related resources. -// Note that the workflow template is loaded from the database/k8s, so workflow.WorkflowTemplate.Manifest is not used. -// Required: -// * workflow.Parameters -// * workflow.Labels (optional) +// If workflow.Name is set, it is used instead of a generated name. +// If there is a parameter named "workflow-execution-name" in workflow.Parameters, it is set as the name. func (c *Client) CreateWorkflowExecution(namespace string, workflow *WorkflowExecution, workflowTemplate *WorkflowTemplate) (*WorkflowExecution, error) { opts := &WorkflowExecutionOptions{ Labels: make(map[string]string), Parameters: workflow.Parameters, } + if workflow.Name != "" { + opts.Name = workflow.Name + } + + if workflowExecutionName := workflow.GetParameterValue("workflow-execution-name"); workflowExecutionName != nil { + opts.Name = *workflowExecutionName + } + nameUID, err := uid2.GenerateUID(workflowTemplate.Name, 63) if err != nil { return nil, err @@ -418,19 +453,16 @@ func (c *Client) CreateWorkflowExecution(namespace string, workflow *WorkflowExe opts.Labels[workflowTemplateVersionLabelKey] = fmt.Sprint(workflowTemplate.Version) label.MergeLabelsPrefix(opts.Labels, workflow.Labels, label.TagPrefix) - // @todo we need to enforce the below requirement in API. - //UX will prevent multiple workflows - manifest, err := workflowTemplate.GetWorkflowManifestBytes() + workflows, err := getWorkflowsFromWorkflowTemplate(workflowTemplate) if err != nil { - log.WithFields(log.Fields{ - "Namespace": namespace, - "WorkflowTemplate": workflowTemplate, - "Error": err.Error(), - }).Error("Error with getting WorkflowManifest from workflow template") return nil, err } - workflows, err := UnmarshalWorkflows(manifest, true) + if len(workflows) != 1 { + return nil, fmt.Errorf("workflow Template contained more than 1 workflow execution") + } + + createdWorkflow, err := c.createWorkflow(namespace, workflowTemplate.ID, workflowTemplate.WorkflowTemplateVersionID, &workflows[0], opts) if err != nil { log.WithFields(log.Fields{ "Namespace": namespace, @@ -440,35 +472,14 @@ func (c *Client) CreateWorkflowExecution(namespace string, workflow *WorkflowExe return nil, err } - id, createdWorkflow, err := c.createWorkflow(namespace, workflowTemplate.ID, workflowTemplate.WorkflowTemplateVersionID, &workflows[0], opts) - if err != nil { - log.WithFields(log.Fields{ - "Namespace": namespace, - "Workflow": workflow, - "Error": err.Error(), - }).Error("Error parsing workflow.") + if _, err := c.InsertLabels(TypeWorkflowExecution, createdWorkflow.ID, workflow.Labels); err != nil { return nil, err } - if _, err := c.InsertLabels(TypeWorkflowExecution, id, workflow.Labels); err != nil { - return nil, err - } - - if createdWorkflow == nil { - err = errors.New("unable to create workflow") - log.WithFields(log.Fields{ - "Namespace": namespace, - "WorkflowTemplate": workflowTemplate, - "Error": err.Error(), - }).Error("Error parsing workflow.") - - return nil, err - } - - workflow.ID = id + workflow.ID = createdWorkflow.ID workflow.Name = createdWorkflow.Name - workflow.CreatedAt = createdWorkflow.CreationTimestamp.UTC() - workflow.UID = createdWorkflow.Name + workflow.CreatedAt = createdWorkflow.CreatedAt.UTC() + workflow.UID = createdWorkflow.UID workflow.WorkflowTemplate = workflowTemplate return workflow, nil @@ -494,44 +505,41 @@ func (c *Client) CloneWorkflowExecution(namespace, uid string) (*WorkflowExecuti return c.CreateWorkflowExecution(namespace, workflowExecution, workflowTemplate) } -func (c *Client) insertPreWorkflowExecutionStatistic(namespace, name string, workflowTemplateVersionId uint64, createdAt time.Time, uid string, parameters []Parameter) (newId uint64, err error) { - tx, err := c.DB.Begin() +// createWorkflowExecutionDB inserts a workflow execution into the database. +// Required fields +// * name +// * createdAt // we sync the argo created at with the db +// * parameters, if any +// * WorkflowTemplate.WorkflowTemplateVersionID +// +// After success, the passed in WorkflowExecution will have it's ID set to the new db record. +func (c *Client) createWorkflowExecutionDB(namespace string, workflowExecution *WorkflowExecution) (err error) { + parametersJSON, err := json.Marshal(workflowExecution.Parameters) if err != nil { - return 0, err - } - defer tx.Rollback() - - parametersJSON, err := json.Marshal(parameters) - if err != nil { - return 0, err + return err } - insertMap := sq.Eq{ - "uid": uid, - "workflow_template_version_id": workflowTemplateVersionId, - "name": name, - "namespace": namespace, - "created_at": createdAt.UTC(), - "phase": wfv1.NodePending, - "parameters": string(parametersJSON), - "is_archived": false, + if err := workflowExecution.GenerateUID(workflowExecution.Name); err != nil { + return err } err = sb.Insert("workflow_executions"). - SetMap(insertMap). + SetMap(sq.Eq{ + "UID": workflowExecution.UID, + "workflow_template_version_id": workflowExecution.WorkflowTemplate.WorkflowTemplateVersionID, + "name": workflowExecution.Name, + "namespace": namespace, + "created_at": workflowExecution.CreatedAt.UTC(), + "phase": wfv1.NodePending, + "parameters": string(parametersJSON), + "is_archived": false, + }). Suffix("RETURNING id"). - RunWith(tx). + RunWith(c.DB). QueryRow(). - Scan(&newId) + Scan(&workflowExecution.ID) - if err != nil { - return 0, err - } - err = tx.Commit() - if err != nil { - return 0, err - } - return newId, err + return } func (c *Client) FinishWorkflowExecutionStatisticViaExitHandler(namespace, name string, workflowTemplateID int64, phase wfv1.NodePhase, startedAt time.Time) (err error) { @@ -562,32 +570,20 @@ func (c *Client) FinishWorkflowExecutionStatisticViaExitHandler(namespace, name } func (c *Client) CronStartWorkflowExecutionStatisticInsert(namespace, uid string, workflowTemplateID int64) (err error) { - query, args, err := c.workflowTemplatesSelectBuilder(namespace). + queryWt := c.workflowTemplatesSelectBuilder(namespace). Where(sq.Eq{ "wt.id": workflowTemplateID, - }). - ToSql() - if err != nil { - return err - } + }) workflowTemplate := &WorkflowTemplate{} - if err := c.DB.Get(workflowTemplate, query, args...); err != nil { + if err := c.DB.Getx(workflowTemplate, queryWt); err != nil { return err } - query, args, err = c.cronWorkflowSelectBuilder(namespace, workflowTemplate.UID).ToSql() - if err != nil { - return err - } + queryCw := c.cronWorkflowSelectBuilder(namespace, workflowTemplate.UID) cronWorkflow := &CronWorkflow{} - if err := c.DB.Get(cronWorkflow, query, args...); err != nil { - return err - } - - cronLabels, err := c.GetDbLabels(TypeCronWorkflow, cronWorkflow.ID) - if err != nil { + if err := c.DB.Getx(cronWorkflow, queryCw); err != nil { return err } @@ -602,49 +598,43 @@ func (c *Client) CronStartWorkflowExecutionStatisticInsert(namespace, uid string return err } - insertMap := sq.Eq{ - "uid": uid, - "workflow_template_version_id": cronWorkflow.WorkflowTemplateVersionID, - "name": uid, - "namespace": namespace, - "phase": wfv1.NodeRunning, - "started_at": time.Now().UTC(), - "cron_workflow_id": cronWorkflow.ID, - "parameters": string(parametersJSON), - } - - workflowExecutionId := uint64(0) + workflowExecutionID := uint64(0) err = sb.Insert("workflow_executions"). - SetMap(insertMap). + SetMap(sq.Eq{ + "uid": uid, + "workflow_template_version_id": cronWorkflow.WorkflowTemplateVersionID, + "name": uid, + "namespace": namespace, + "phase": wfv1.NodeRunning, + "started_at": time.Now().UTC(), + "cron_workflow_id": cronWorkflow.ID, + "parameters": string(parametersJSON), + }). Suffix("RETURNING id"). RunWith(tx). QueryRow(). - Scan(&workflowExecutionId) + Scan(&workflowExecutionID) if err != nil { return err } - if len(cronLabels) > 0 { - labelsMapped := LabelsToMapping(cronLabels...) - _, err = c.InsertLabelsBuilder(TypeWorkflowExecution, workflowExecutionId, labelsMapped). - RunWith(tx). - Exec() - if err != nil { - return err - } + cronLabels, err := c.GetDBLabelsMapped(TypeCronWorkflow, cronWorkflow.ID) + if err != nil { + return err + } + labelsMapped := cronLabels[cronWorkflow.ID] + if _, err := c.InsertLabelsRunner(tx, TypeWorkflowExecution, workflowExecutionID, labelsMapped); err != nil { + return err } err = tx.Commit() - if err != nil { - return err - } + return err } func (c *Client) GetWorkflowExecution(namespace, uid string) (workflow *WorkflowExecution, err error) { workflow = &WorkflowExecution{} - - query, args, err := sb.Select(getWorkflowExecutionColumns("we", "")...). + query := sb.Select(getWorkflowExecutionColumns("we")...). Columns(getWorkflowTemplateColumns("wt", "workflow_template")...). Columns(`wtv.manifest "workflow_template.manifest"`). From("workflow_executions we"). @@ -654,12 +644,13 @@ func (c *Client) GetWorkflowExecution(namespace, uid string) (workflow *Workflow "wt.namespace": namespace, "we.name": uid, "we.is_archived": false, - }). - ToSql() - if err != nil { - return nil, err - } - if err := c.DB.Get(workflow, query, args...); err != nil { + }) + + if err := c.DB.Getx(workflow, query); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, err } @@ -677,7 +668,7 @@ func (c *Client) GetWorkflowExecution(namespace, uid string) (workflow *Workflow version, err := strconv.ParseInt( wf.ObjectMeta.Labels[workflowTemplateVersionLabelKey], 10, - 32, + 64, ) if err != nil { log.WithFields(log.Fields{ @@ -713,6 +704,7 @@ func (c *Client) GetWorkflowExecution(namespace, uid string) (workflow *Workflow return } +// ListWorkflowExecutions gets a list of WorkflowExecutions ordered by most recently created first. func (c *Client) ListWorkflowExecutions(namespace, workflowTemplateUID, workflowTemplateVersion string, paginator *pagination.PaginationRequest) (workflows []*WorkflowExecution, err error) { sb := workflowExecutionsSelectBuilder(namespace, workflowTemplateUID, workflowTemplateVersion). OrderBy("we.created_at DESC") @@ -725,10 +717,11 @@ func (c *Client) ListWorkflowExecutions(namespace, workflowTemplateUID, workflow return } +// CountWorkflowExecutions returns the number of workflow executions func (c *Client) CountWorkflowExecutions(namespace, workflowTemplateUID, workflowTemplateVersion string) (count int, err error) { err = workflowExecutionsSelectBuilderNoColumns(namespace, workflowTemplateUID, workflowTemplateVersion). Columns("COUNT(*)"). - RunWith(c.DB.DB). + RunWith(c.DB). QueryRow(). Scan(&count) @@ -1069,8 +1062,9 @@ func (c *Client) SuspendWorkflowExecution(namespace, uid string) (err error) { return } +// TerminateWorkflowExecution marks a workflows execution as terminated in DB and terminates the argo resource. func (c *Client) TerminateWorkflowExecution(namespace, uid string) (err error) { - query, args, err := sb.Update("workflow_executions"). + _, err = sb.Update("workflow_executions"). Set("phase", "Terminated"). Set("started_at", time.Time.UTC(time.Now())). Set("finished_at", time.Time.UTC(time.Now())). @@ -1078,16 +1072,12 @@ func (c *Client) TerminateWorkflowExecution(namespace, uid string) (err error) { "uid": uid, "namespace": namespace, }). - ToSql() - + RunWith(c.DB). + Exec() if err != nil { return err } - if _, err := c.DB.Exec(query, args...); err != nil { - return err - } - err = argoutil.TerminateWorkflow(c.ArgoprojV1alpha1().Workflows(namespace), uid) return @@ -1605,7 +1595,7 @@ func workflowExecutionsSelectBuilder(namespace, workflowTemplateUID, workflowTem } func (c *Client) getWorkflowExecutionAndTemplate(namespace string, uid string) (workflow *WorkflowExecution, err error) { - query, args, err := sb.Select(getWorkflowExecutionColumns("we", "")...). + sb := sb.Select(getWorkflowExecutionColumns("we", "")...). Columns(getWorkflowTemplateColumns("wt", "workflow_template")...). Columns(`wtv.manifest "workflow_template.manifest"`, `wtv.version "workflow_template.version"`). From("workflow_executions we"). @@ -1615,16 +1605,10 @@ func (c *Client) getWorkflowExecutionAndTemplate(namespace string, uid string) ( "wt.namespace": namespace, "we.name": uid, "we.is_archived": false, - }). - ToSql() - if err != nil { - return nil, err - } - - // TODO DB call + }) workflow = &WorkflowExecution{} - if err = c.DB.Get(workflow, query, args...); err != nil { + if err = c.DB.Getx(workflow, sb); err != nil { return nil, err } diff --git a/pkg/workflow_execution_test.go b/pkg/workflow_execution_test.go new file mode 100644 index 0000000..47760da --- /dev/null +++ b/pkg/workflow_execution_test.go @@ -0,0 +1,99 @@ +package v1 + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +// TestClient_CreateWorkflowExecution tests creating a workflow execution +func TestClient_CreateWorkflowExecution(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + wt := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + wt, _ = c.CreateWorkflowTemplate(namespace, wt) + + we := &WorkflowExecution{ + Name: "test", + } + + we, err := c.CreateWorkflowExecution(namespace, we, wt) + assert.Nil(t, err) +} + +// TestClient_GetWorkflowExecution tests getting a workflow execution that exists +func TestClient_GetWorkflowExecution(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + wt := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + wt, _ = c.CreateWorkflowTemplate(namespace, wt) + + we := &WorkflowExecution{ + Name: "test", + } + + we, _ = c.CreateWorkflowExecution(namespace, we, wt) + + getWe, err := c.GetWorkflowExecution(namespace, we.UID) + assert.Nil(t, err) + + assert.Equal(t, we.Name, getWe.Name) + assert.Equal(t, we.UID, getWe.UID) +} + +// TestClient_GetWorkflowExecution tests getting a workflow execution that doesn't exist +func TestClient_GetWorkflowExecution_NotExists(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + getWe, err := c.GetWorkflowExecution(namespace, "not-exist") + assert.Nil(t, getWe) + assert.Nil(t, err) +} + +// TestClient_ArchiveWorkflowExecution_NotExist makes sure there is no error if the workflow +// execution does not exist +func TestClient_ArchiveWorkflowExecution_NotExist(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + err := c.ArchiveWorkflowExecution("onepanel-no-exist", "test-no-exist") + assert.Nil(t, err) +} + +// TestClient_ArchiveWorkflowExecution_Exist makes sure we archive an existing workflow execution correctly +func TestClient_ArchiveWorkflowExecution_Exist(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + weName := "test" + + wt := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + wt, _ = c.CreateWorkflowTemplate(namespace, wt) + + we := &WorkflowExecution{ + Name: weName, + } + + we, err := c.CreateWorkflowExecution(namespace, we, wt) + + err = c.ArchiveWorkflowExecution(namespace, weName) + assert.Nil(t, err) +} diff --git a/pkg/workflow_execution_types.go b/pkg/workflow_execution_types.go index e47236a..08e19a2 100644 --- a/pkg/workflow_execution_types.go +++ b/pkg/workflow_execution_types.go @@ -3,6 +3,7 @@ package v1 import ( "encoding/json" wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" + uid2 "github.com/onepanelio/core/pkg/util/uid" "github.com/onepanelio/core/util/sql" "time" ) @@ -22,6 +23,7 @@ type WorkflowExecution struct { FinishedAt *time.Time `db:"finished_at"` WorkflowTemplate *WorkflowTemplate `db:"workflow_template"` Labels map[string]string + ArgoWorkflow *wfv1.Workflow } // WorkflowExecutionOptions are options you have for an executing workflow @@ -55,6 +57,18 @@ type WorkflowExecutionStatus struct { FinishedAt *time.Time `db:"finished_at" json:"finishedAt"` } +// GenerateUID generates a uid from the input name and sets it on the workflow execution +func (we *WorkflowExecution) GenerateUID(name string) error { + result, err := uid2.GenerateUID(name, 63) + if err != nil { + return err + } + + we.UID = result + + return nil +} + // LoadParametersFromBytes loads Parameters from the WorkflowExecution's ParameterBytes field. func (we *WorkflowExecution) LoadParametersFromBytes() ([]Parameter, error) { loadedParameters := make([]Parameter, 0) @@ -75,6 +89,17 @@ func (we *WorkflowExecution) LoadParametersFromBytes() ([]Parameter, error) { return we.Parameters, err } +// GetParameterValue returns the value of the parameter with the given name, or nil if there is no such parameter +func (we *WorkflowExecution) GetParameterValue(name string) *string { + for _, p := range we.Parameters { + if p.Name == name { + return p.Value + } + } + + return nil +} + // getWorkflowExecutionColumns returns all of the columns for workflowExecution modified by alias, destination. // see formatColumnSelect func getWorkflowExecutionColumns(aliasAndDestination ...string) []string { diff --git a/pkg/workflow_template.go b/pkg/workflow_template.go index a024867..cf9d8e1 100644 --- a/pkg/workflow_template.go +++ b/pkg/workflow_template.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "github.com/onepanelio/core/pkg/util/pagination" - uid2 "github.com/onepanelio/core/pkg/util/uid" "strconv" "strings" "time" @@ -21,23 +20,67 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (c *Client) createWorkflowTemplate(namespace string, workflowTemplate *WorkflowTemplate) (*WorkflowTemplate, *WorkflowTemplateVersion, error) { - uid, err := uid2.GenerateUID(workflowTemplate.Name, 30) +// createWorkflowTemplateVersionDB inserts a record into workflow_template_versions using the current time accurate to nanoseconds +// the data is returned in the resulting WorkflowTemplateVersion struct. +func createWorkflowTemplateVersionDB(runner sq.BaseRunner, workflowTemplateID uint64, manifest string, latest bool) (workflowTemplateVersion *WorkflowTemplateVersion, err error) { + ts := time.Now().UnixNano() + + workflowTemplateVersion = &WorkflowTemplateVersion{ + WorkflowTemplate: &WorkflowTemplate{ + ID: workflowTemplateID, + }, + Manifest: manifest, + IsLatest: latest, + Version: ts, + } + + err = sb.Insert("workflow_template_versions"). + SetMap(sq.Eq{ + "workflow_template_id": workflowTemplateID, + "version": ts, + "is_latest": true, + "manifest": manifest, + }). + Suffix("RETURNING id"). + RunWith(runner). + QueryRow(). + Scan(&workflowTemplateVersion.ID) + + return +} + +// createLatestWorkflowTemplateVersionDB creates a new workflow template version and marks all previous versions as not latest. +func createLatestWorkflowTemplateVersionDB(runner sq.BaseRunner, workflowTemplateID uint64, manifest string) (workflowTemplateVersion *WorkflowTemplateVersion, err error) { + _, err = sb.Update("workflow_template_versions"). + Set("is_latest", false). + Where(sq.Eq{ + "workflow_template_id": workflowTemplateID, + }). + RunWith(runner). + Exec() if err != nil { + return nil, err + } + + return createWorkflowTemplateVersionDB(runner, workflowTemplateID, manifest, true) +} + +// createWorkflowTemplate creates a WorkflowTemplate and all of the DB/Argo/K8s related resources +// The returned WorkflowTemplate has the ArgoWorkflowTemplate set to the newly created one. +func (c *Client) createWorkflowTemplate(namespace string, workflowTemplate *WorkflowTemplate) (*WorkflowTemplate, *WorkflowTemplateVersion, error) { + if err := workflowTemplate.GenerateUID(workflowTemplate.Name); err != nil { return nil, nil, err } - workflowTemplate.UID = uid + tx, err := c.DB.Begin() if err != nil { return nil, nil, err } defer tx.Rollback() - versionUnix := time.Now().Unix() - err = sb.Insert("workflow_templates"). SetMap(sq.Eq{ - "uid": uid, + "uid": workflowTemplate.UID, "name": workflowTemplate.Name, "namespace": namespace, "is_system": workflowTemplate.IsSystem, @@ -50,33 +93,22 @@ func (c *Client) createWorkflowTemplate(namespace string, workflowTemplate *Work return nil, nil, err } - workflowTemplateVersion := &WorkflowTemplateVersion{} - err = sb.Insert("workflow_template_versions"). - SetMap(sq.Eq{ - "workflow_template_id": workflowTemplate.ID, - "version": versionUnix, - "is_latest": true, - "manifest": workflowTemplate.Manifest, - }). - Suffix("RETURNING id"). - RunWith(tx). - QueryRow(). - Scan(&workflowTemplateVersion.ID) + workflowTemplateVersion, err := createWorkflowTemplateVersionDB(tx, workflowTemplate.ID, workflowTemplate.Manifest, true) if err != nil { return nil, nil, err } + workflowTemplate.WorkflowTemplateVersionID = workflowTemplateVersion.ID _, err = c.InsertLabelsRunner(tx, TypeWorkflowTemplateVersion, workflowTemplateVersion.ID, workflowTemplate.Labels) if err != nil { return nil, nil, err } - argoWft, err := createArgoWorkflowTemplate(workflowTemplate, versionUnix) + argoWft, err := createArgoWorkflowTemplate(workflowTemplate, workflowTemplateVersion.Version) if err != nil { return nil, nil, err } - - argoWft.Labels[label.WorkflowTemplateVersionUid] = strconv.FormatInt(versionUnix, 10) + argoWft.Labels[label.WorkflowTemplateVersionUid] = strconv.FormatInt(workflowTemplateVersion.Version, 10) if workflowTemplate.Resource != nil && workflowTemplate.ResourceUID != nil { if *workflowTemplate.Resource == TypeWorkspaceTemplate { @@ -96,7 +128,8 @@ func (c *Client) createWorkflowTemplate(namespace string, workflowTemplate *Work return nil, nil, err } - workflowTemplate.Version = versionUnix + workflowTemplate.ArgoWorkflowTemplate = argoWft + workflowTemplate.Version = workflowTemplateVersion.Version return workflowTemplate, workflowTemplateVersion, nil } @@ -131,6 +164,8 @@ func (c *Client) countWorkflowTemplateSelectBuilder(namespace string) sq.SelectB return sb } +// workflowTemplatesVersionSelectBuilder selects data from workflow template versions joined to a workflow template +// the versions/template are filtered by the workflow template's namespace. func (c *Client) workflowTemplatesVersionSelectBuilder(namespace string) sq.SelectBuilder { sb := sb.Select(getWorkflowTemplateVersionColumns("wtv")...). From("workflow_template_versions wtv"). @@ -142,11 +177,14 @@ func (c *Client) workflowTemplatesVersionSelectBuilder(namespace string) sq.Sele return sb } -// GetWorkflowTemplateDB returns a WorkflowTemplate from the database -func (c *Client) GetWorkflowTemplateDB(namespace, name string) (workflowTemplate *WorkflowTemplate, err error) { +// GetWorkflowTemplateDB returns a WorkflowTemplate from the database that is not archived, should one exist. +func (c *Client) getWorkflowTemplateDB(namespace, name string) (workflowTemplate *WorkflowTemplate, err error) { + workflowTemplate = &WorkflowTemplate{} + sb := c.workflowTemplatesSelectBuilder(namespace). Where(sq.Eq{ - "name": name, + "wt.name": name, + "wt.is_archived": false, }) err = c.DB.Getx(workflowTemplate, sb) @@ -154,9 +192,11 @@ func (c *Client) GetWorkflowTemplateDB(namespace, name string) (workflowTemplate return } -// GetWorkflowTemplateVersionDB will return a WorkflowTemplateVersion given the arguments. +// getWorkflowTemplateVersionDB will return a WorkflowTemplateVersion given the arguments. // version can be a number as a string, or the string "latest" to get the latest. -func (c *Client) GetWorkflowTemplateVersionDB(namespace, name, version string) (workflowTemplateVersion *WorkflowTemplateVersion, err error) { +func (c *Client) getWorkflowTemplateVersionDB(namespace, name, version string) (workflowTemplateVersion *WorkflowTemplateVersion, err error) { + workflowTemplateVersion = &WorkflowTemplateVersion{} + whereMap := sq.Eq{ "wt.name": name, } @@ -176,8 +216,8 @@ func (c *Client) GetWorkflowTemplateVersionDB(namespace, name, version string) ( } // GetLatestWorkflowTemplateVersionDB returns the latest WorkflowTemplateVersion -func (c *Client) GetLatestWorkflowTemplateVersionDB(namespace, name string) (workflowTemplateVersion *WorkflowTemplateVersion, err error) { - return c.GetWorkflowTemplateVersionDB(namespace, name, "latest") +func (c *Client) getLatestWorkflowTemplateVersionDB(namespace, name string) (workflowTemplateVersion *WorkflowTemplateVersion, err error) { + return c.getWorkflowTemplateVersionDB(namespace, name, "latest") } func (c *Client) getWorkflowTemplateById(id uint64) (workflowTemplate *WorkflowTemplate, err error) { @@ -197,13 +237,16 @@ func (c *Client) getWorkflowTemplateById(id uint64) (workflowTemplate *WorkflowT return } -// If version is 0, the latest workflow template is fetched. +// getWorkflowTemplate gets the workflowtemplate given the input data. +// it also loads the argo workflow and labels data. +// If version is <= 0, the latest workflow template is fetched. +// If not found, (nil, nil) is returned func (c *Client) getWorkflowTemplate(namespace, uid string, version int64) (workflowTemplate *WorkflowTemplate, err error) { workflowTemplate = &WorkflowTemplate{ WorkflowExecutionStatisticReport: &WorkflowExecutionStatisticReport{}, } - // A new workflow template version is created upon a change, so we use it's createdAt + // A new workflow template version is created upon a change, so we use it's created_at // as a modified_at for the workflow template. sb := c.workflowTemplatesSelectBuilder(namespace). Columns("wtv.manifest", "wtv.version", "wtv.id workflow_template_version_id", "wtv.created_at modified_at"). @@ -213,7 +256,7 @@ func (c *Client) getWorkflowTemplate(namespace, uid string, version int64) (work "wt.is_archived": false, }) - if version == 0 { + if version <= 0 { sb = sb.Where(sq.Eq{"wtv.is_latest": true}) } else { sb = sb.Where(sq.Eq{"wtv.version": version}) @@ -256,7 +299,7 @@ func (c *Client) getWorkflowTemplate(namespace, uid string, version int64) (work } func (c *Client) listWorkflowTemplateVersions(namespace, uid string) (workflowTemplateVersions []*WorkflowTemplate, err error) { - dbVersions, err := c.listDBWorkflowTemplateVersions(namespace, uid) + dbVersions, err := c.selectWorkflowTemplateVersionsDB(namespace, uid) if err != nil { return nil, err } @@ -291,8 +334,10 @@ func (c *Client) listWorkflowTemplateVersions(namespace, uid string) (workflowTe return } -func (c *Client) listWorkflowTemplates(namespace string, paginator *pagination.PaginationRequest) (workflowTemplateVersions []*WorkflowTemplate, err error) { - workflowTemplateVersions = []*WorkflowTemplate{} +// selectWorkflowTemplatesDB loads workflow templates from the database for the input namespace +// it also selects the total number of versions and latest version id +func (c *Client) selectWorkflowTemplatesDB(namespace string, paginator *pagination.PaginationRequest) (workflowTemplates []*WorkflowTemplate, err error) { + workflowTemplates = make([]*WorkflowTemplate, 0) sb := c.workflowTemplatesSelectBuilder(namespace). Column("COUNT(wtv.*) versions, MAX(wtv.id) workflow_template_version_id"). @@ -303,28 +348,24 @@ func (c *Client) listWorkflowTemplates(namespace string, paginator *pagination.P "wt.is_system": false, }). OrderBy("wt.created_at DESC") - sb = *paginator.ApplyToSelect(&sb) - query, args, err := sb.ToSql() - if err != nil { - return - } - err = c.DB.Select(&workflowTemplateVersions, query, args...) + err = c.DB.Selectx(&workflowTemplates, sb) return } +// CountWorkflowTemplates counts the total number of workflow templates for the given namespace +// archived, and system templates are ignored. func (c *Client) CountWorkflowTemplates(namespace string) (count int, err error) { - err = sb.Select("COUNT( DISTINCT( wt.id ))"). + err = sb.Select("COUNT(*)"). From("workflow_templates wt"). - Join("workflow_template_versions wtv ON wtv.workflow_template_id = wt.id"). Where(sq.Eq{ "wt.namespace": namespace, "wt.is_archived": false, "wt.is_system": false, }). - RunWith(c.DB.DB). + RunWith(c.DB). QueryRow(). Scan(&count) @@ -368,6 +409,11 @@ func (c *Client) CreateWorkflowTemplate(namespace string, workflowTemplate *Work return newWorkflowTemplate, nil } +// CreateWorkflowTemplateVersion creates a new workflow template version including argo resources. +// It marks any older workflow template versions as not latest +// +// Pre-condition: a Workflow Template version already exists +// Post-condition: the input workflow template will have it's fields updated so it matches the new version data. func (c *Client) CreateWorkflowTemplateVersion(namespace string, workflowTemplate *WorkflowTemplate) (*WorkflowTemplate, error) { if workflowTemplate.UID == "" { return nil, fmt.Errorf("uid required for CreateWorkflowTemplateVersion") @@ -378,8 +424,6 @@ func (c *Client) CreateWorkflowTemplateVersion(namespace string, workflowTemplat return nil, util.NewUserError(codes.InvalidArgument, err.Error()) } - versionUnix := time.Now().Unix() - tx, err := c.DB.Begin() if err != nil { return nil, err @@ -391,75 +435,26 @@ func (c *Client) CreateWorkflowTemplateVersion(namespace string, workflowTemplat "wt.uid": workflowTemplate.UID, "wt.is_archived": false, }) - query, args, err := wftSb.ToSql() - if err != nil { - return nil, err - } - workflowTemplateDb := &WorkflowTemplate{} - if err = c.DB.Get(workflowTemplateDb, query, args...); err != nil { + workflowTemplateDB := &WorkflowTemplate{} + if err = c.DB.Getx(workflowTemplateDB, wftSb); err != nil { return nil, err } - _, err = sb.Update("workflow_template_versions"). - Set("is_latest", false). - Where(sq.Eq{ - "workflow_template_id": workflowTemplateDb.ID, - }). - RunWith(tx). - Exec() + workflowTemplateVersion, err := createLatestWorkflowTemplateVersionDB(tx, workflowTemplateDB.ID, workflowTemplate.Manifest) if err != nil { return nil, err } + workflowTemplate.WorkflowTemplateVersionID = workflowTemplateVersion.ID - workflowTemplateVersionID := uint64(0) - err = sb.Insert("workflow_template_versions"). - SetMap(sq.Eq{ - "workflow_template_id": workflowTemplateDb.ID, - "version": versionUnix, - "is_latest": true, - "manifest": workflowTemplate.Manifest, - }). - Suffix("RETURNING id"). - RunWith(tx). - QueryRow(). - Scan(&workflowTemplateVersionID) + updatedTemplate, err := createArgoWorkflowTemplate(workflowTemplate, workflowTemplateVersion.Version) if err != nil { return nil, err } - workflowTemplate.WorkflowTemplateVersionID = workflowTemplateVersionID - latest, err := c.getArgoWorkflowTemplate(namespace, workflowTemplate.UID, "latest") - if err != nil { - log.WithFields(log.Fields{ - "Namespace": namespace, - "WorkflowTemplate": workflowTemplate, - "Error": err.Error(), - }).Error("Could not get latest argo workflow template") - - return nil, err - } - - delete(latest.Labels, label.VersionLatest) - - latest, err = c.ArgoprojV1alpha1().WorkflowTemplates(namespace).Update(latest) - if err != nil { - return nil, err - } - - updatedTemplate, err := createArgoWorkflowTemplate(workflowTemplate, versionUnix) - if err != nil { - latest.Labels[label.VersionLatest] = "true" - if _, err := c.ArgoprojV1alpha1().WorkflowTemplates(namespace).Update(latest); err != nil { - return nil, err - } - - return nil, err - } - updatedTemplate.TypeMeta = v1.TypeMeta{} updatedTemplate.ObjectMeta.ResourceVersion = "" updatedTemplate.ObjectMeta.SetSelfLink("") - updatedTemplate.Labels[label.WorkflowTemplateVersionUid] = strconv.FormatInt(versionUnix, 10) + updatedTemplate.Labels[label.WorkflowTemplateVersionUid] = strconv.FormatInt(workflowTemplateVersion.Version, 10) parametersMap, err := workflowTemplate.GetParametersKeyString() if err != nil { @@ -473,15 +468,32 @@ func (c *Client) CreateWorkflowTemplateVersion(namespace string, workflowTemplat updatedTemplate.Annotations[key] = value } + latest, err := c.getArgoWorkflowTemplate(namespace, workflowTemplate.UID, "latest") + if err != nil { + log.WithFields(log.Fields{ + "Namespace": namespace, + "WorkflowTemplate": workflowTemplate, + "Error": err.Error(), + }).Error("Could not get latest argo workflow template") + + return nil, err + } + delete(latest.Labels, label.VersionLatest) + if _, err := c.ArgoprojV1alpha1().WorkflowTemplates(namespace).Create(updatedTemplate); err != nil { return nil, err } + latest, err = c.ArgoprojV1alpha1().WorkflowTemplates(namespace).Update(latest) + if err != nil { + return nil, err + } + if err := tx.Commit(); err != nil { return nil, err } - workflowTemplate.Version = versionUnix + workflowTemplate.Version = workflowTemplateVersion.Version return workflowTemplate, nil } @@ -564,7 +576,7 @@ func (c *Client) ListWorkflowTemplateVersions(namespace, uid string) (workflowTe } func (c *Client) ListWorkflowTemplates(namespace string, paginator *pagination.PaginationRequest) (workflowTemplateVersions []*WorkflowTemplate, err error) { - workflowTemplateVersions, err = c.listWorkflowTemplates(namespace, paginator) + workflowTemplateVersions, err = c.selectWorkflowTemplatesDB(namespace, paginator) if err != nil { log.WithFields(log.Fields{ "Namespace": namespace, @@ -722,6 +734,8 @@ func (c *Client) ArchiveWorkflowTemplate(namespace, uid string) (archived bool, return true, nil } +// createArgoWorkflowTemplate creates an argo workflow template from the workflowTemplate struct +// the argo template stores the version information. func createArgoWorkflowTemplate(workflowTemplate *WorkflowTemplate, version int64) (*v1alpha1.WorkflowTemplate, error) { var argoWft *v1alpha1.WorkflowTemplate var jsonOpts []argojson.JSONOpt @@ -737,15 +751,14 @@ func createArgoWorkflowTemplate(workflowTemplate *WorkflowTemplate, version int6 return nil, err } - worfklowTemplateName, err := uid2.GenerateUID(workflowTemplate.Name, 30) - if err != nil { + if err := workflowTemplate.GenerateUID(workflowTemplate.Name); err != nil { return nil, err } - argoWft.Name = fmt.Sprintf("%v-v%v", worfklowTemplateName, version) + argoWft.Name = fmt.Sprintf("%v-v%v", workflowTemplate.UID, version) labels := map[string]string{ - label.WorkflowTemplate: worfklowTemplateName, + label.WorkflowTemplate: workflowTemplate.UID, label.WorkflowTemplateUid: workflowTemplate.UID, label.Version: fmt.Sprintf("%v", version), label.VersionLatest: "true", @@ -757,9 +770,10 @@ func createArgoWorkflowTemplate(workflowTemplate *WorkflowTemplate, version int6 return argoWft, nil } -// version "latest" will get the latest version. -func (c *Client) getArgoWorkflowTemplate(namespace, workflowTemplateUid, version string) (*v1alpha1.WorkflowTemplate, error) { - labelSelect := fmt.Sprintf("%v=%v", label.WorkflowTemplateUid, workflowTemplateUid) +// getArgoWorkflowTemplate will load the argo workflow template. +// version "latest" will get the latest version, otherwise a number (as a string) will be used. +func (c *Client) getArgoWorkflowTemplate(namespace, workflowTemplateUID, version string) (*v1alpha1.WorkflowTemplate, error) { + labelSelect := fmt.Sprintf("%v=%v", label.WorkflowTemplateUid, workflowTemplateUID) if version == "latest" { labelSelect += "," + label.VersionLatest + "=true" } else { @@ -799,28 +813,22 @@ func (c *Client) listArgoWorkflowTemplates(namespace, workflowTemplateUid string return &templates, nil } -func (c *Client) listDBWorkflowTemplateVersions(namespace, workflowTemplateUID string) ([]*WorkflowTemplateVersion, error) { - versions := make([]*WorkflowTemplateVersion, 0) +// listDBWorkflowTemplateVersions gets all of the workflow template versions for a specified workflow template uid +// archived ones are ignored. Returned in created_at desc order. +func (c *Client) selectWorkflowTemplateVersionsDB(namespace, workflowTemplateUID string) (versions []*WorkflowTemplateVersion, err error) { + versions = make([]*WorkflowTemplateVersion, 0) sb := c.workflowTemplatesVersionSelectBuilder(namespace). - Columns(`wt.id "workflow_template.id"`, `wt.created_at "workflow_template.created_at"`). - Columns(`wt.name "workflow_template.name"`, `wt.is_archived "workflow_template.is_archived"`). + Columns(getWorkflowTemplateColumns("wt", "workflow_template")...). Where(sq.Eq{ "wt.uid": workflowTemplateUID, "wt.is_archived": false, }). OrderBy("wtv.created_at DESC") - query, args, err := sb.ToSql() - if err != nil { - return versions, err - } + err = c.DB.Selectx(&versions, sb) - if err := c.DB.Select(&versions, query, args...); err != nil { - return versions, err - } - - return versions, nil + return } // prefix is the label prefix. diff --git a/pkg/workflow_template_test.go b/pkg/workflow_template_test.go new file mode 100644 index 0000000..3aa82db --- /dev/null +++ b/pkg/workflow_template_test.go @@ -0,0 +1,345 @@ +package v1 + +import ( + "database/sql" + "fmt" + "github.com/onepanelio/core/pkg/util" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "testing" +) + +const defaultWorkflowTemplate = `entrypoint: main +arguments: + parameters: + - name: source + value: https://github.com/onepanelio/pytorch-examples.git + - name: command + value: "python mnist/main.py --epochs=1" +volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 2Gi + - metadata: + name: output + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 2Gi +templates: + - name: main + dag: + tasks: + - name: train-model + template: pytorch +# Uncomment section below to send metrics to Slack +# - name: notify-in-slack +# dependencies: [train-model] +# template: slack-notify-success +# arguments: +# parameters: +# - name: status +# value: "{{tasks.train-model.status}}" +# artifacts: +# - name: metrics +# from: "{{tasks.train-model.outputs.artifacts.sys-metrics}}" + - name: pytorch + inputs: + artifacts: + - name: src + path: /mnt/src + git: + repo: "{{workflow.parameters.source}}" + outputs: + artifacts: + - name: model + path: /mnt/output + optional: true + archive: + none: {} + container: + image: pytorch/pytorch:latest + command: [sh,-c] + args: ["{{workflow.parameters.command}}"] + workingDir: /mnt/src + volumeMounts: + - name: data + mountPath: /mnt/data + - name: output + mountPath: /mnt/output + - name: slack-notify-success + container: + image: technosophos/slack-notify + command: [sh,-c] + args: ['SLACK_USERNAME=Worker SLACK_TITLE="{{workflow.name}} {{inputs.parameters.status}}" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE=$(cat /tmp/metrics.json)} ./slack-notify'] + inputs: + parameters: + - name: status + artifacts: + - name: metrics + path: /tmp/metrics.json + optional: true +` + +// testClientGetWorkflowTemplateDBEmpty attempts to get a WorkflowTemplate when there isn't one. +// this should fail. +func testClientGetWorkflowTemplateDBEmpty(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + _, err := c.getWorkflowTemplateDB("test", "test") + assert.Equal(t, sql.ErrNoRows, err) +} + +// testClientGetWorkflowTemplateDBExists gets a WorkflowTemplate when there is one +// this should succeed +func testClientGetWorkflowTemplateDBExists(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + workflowTemplate := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + _, err := c.CreateWorkflowTemplate("onepanel", workflowTemplate) + assert.Nil(t, err) + + _, err = c.getWorkflowTemplateDB("onepanel", "test") + assert.Nil(t, err) +} + +// TestClient_getWorkflowTemplateDB tests getting a workflow template from the database +func TestClient_getWorkflowTemplateDB(t *testing.T) { + testClientGetWorkflowTemplateDBEmpty(t) + testClientGetWorkflowTemplateDBExists(t) +} + +// testClientCreateWorkflowTemplateSuccess makes sure a correct workflow template is created correctly +func testClientCreateWorkflowTemplateSuccess(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + workflowTemplate := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + + wft, err := c.CreateWorkflowTemplate("onepanel", workflowTemplate) + assert.Nil(t, err) + assert.NotNil(t, wft.ArgoWorkflowTemplate) +} + +// testClientCreateWorkflowTemplateTimestamp makes sure we can create mulitple +// workflow templtate versions one after another with practically no time delay. +// This handles an edge case where versions were set using second time precision and could fail in migrations +// as they were created one after another. +func testClientCreateWorkflowTemplateTimestamp(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + workflowTemplate := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + + // This method creates a workflow template version underneath + wft, err := c.CreateWorkflowTemplate("onepanel", workflowTemplate) + assert.Nil(t, err) + assert.NotNil(t, wft.ArgoWorkflowTemplate) + + // This method creates a brand new version + wft, err = c.CreateWorkflowTemplateVersion(namespace, workflowTemplate) + assert.Nil(t, err) + assert.NotNil(t, wft.ArgoWorkflowTemplate) +} + +// testClientCreateWorkflowTemplateInsertSameName attempts to insert a WorkflowTemplate with the same name +// this should fail +func testClientCreateWorkflowTemplateInsertSameName(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + workflowTemplate := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + _, err := c.CreateWorkflowTemplate("onepanel", workflowTemplate) + assert.Nil(t, err) + + _, err = c.CreateWorkflowTemplate("onepanel", workflowTemplate) + assert.NotNil(t, err) + + assert.IsType(t, &util.UserError{}, err) + userErr := err.(*util.UserError) + + assert.Equal(t, userErr.Code, codes.AlreadyExists) +} + +// TestClient_CreateWorkflowTemplate tests creating a workflow template +func TestClient_CreateWorkflowTemplate(t *testing.T) { + testClientCreateWorkflowTemplateInsertSameName(t) + testClientCreateWorkflowTemplateSuccess(t) + testClientCreateWorkflowTemplateTimestamp(t) +} + +// testClientPrivateGetWorkflowTemplateSuccess gets a workflow template with no error conditions encountered +func testClientPrivateGetWorkflowTemplateSuccess(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + workflowTemplate := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + created, _ := c.CreateWorkflowTemplate(namespace, workflowTemplate) + + wt, err := c.getWorkflowTemplate(namespace, created.UID, 0) + assert.NotNil(t, wt) + assert.Nil(t, err) +} + +// testClientGetWorkflowTemplateSuccessVersion gets a workflow template for a specific version with no error conditions encountered +func testClientPrivateGetWorkflowTemplateSuccessVersion(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + workflowTemplate := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + created, _ := c.CreateWorkflowTemplate(namespace, workflowTemplate) + c.CreateWorkflowTemplateVersion(namespace, workflowTemplate) + + wt, err := c.getWorkflowTemplate(namespace, created.UID, created.Version) + assert.NotNil(t, wt) + assert.Nil(t, err) + + assert.Equal(t, created.Version, wt.Version) + assert.Equal(t, created.Manifest, wt.Manifest) +} + +// testClientGetWorkflowTemplateNotFound attempts to get a not-found workflow template +func testClientPrivateGetWorkflowTemplateNotFound(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + wt, err := c.getWorkflowTemplate("onepanel", "uid-not-found", 0) + assert.Nil(t, wt) + assert.Nil(t, err) +} + +// Test_getWorkflowTemplate tests getting a workflow template +func Test_getWorkflowTemplate(t *testing.T) { + testClientPrivateGetWorkflowTemplateSuccess(t) + testClientPrivateGetWorkflowTemplateNotFound(t) + testClientPrivateGetWorkflowTemplateSuccessVersion(t) +} + +func TestClient_getWorkflowTemplateVersionDB(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + name := "test" + workflowTemplate := &WorkflowTemplate{ + Name: name, + Manifest: defaultWorkflowTemplate, + } + + original, _ := c.CreateWorkflowTemplate(namespace, workflowTemplate) + + versionAsString := fmt.Sprintf("%v", original.Version) + originalRes, err := c.getWorkflowTemplateVersionDB(namespace, name, versionAsString) + + assert.Nil(t, err) + assert.Equal(t, original.Version, originalRes.Version) +} + +// testClientCreateWorkflowTemplateVersionNew makes sure you can successfully create a new workflow template version +func testClientCreateWorkflowTemplateVersionNew(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + workflowTemplate := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + + c.CreateWorkflowTemplate(namespace, workflowTemplate) + _, err := c.CreateWorkflowTemplateVersion(namespace, workflowTemplate) + + assert.Nil(t, err) +} + +// testClientCreateWorkflowTemplateVersionMarkOldNotLatest makes sure older versions are no longer marked as latest +func testClientCreateWorkflowTemplateVersionMarkOldNotLatest(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + name := "test" + workflowTemplate := &WorkflowTemplate{ + Name: name, + Manifest: defaultWorkflowTemplate, + } + + original, _ := c.CreateWorkflowTemplate(namespace, workflowTemplate) + originalVersionAsString := fmt.Sprintf("%v", original.Version) + c.CreateWorkflowTemplateVersion(namespace, workflowTemplate) + + updated, _ := c.getWorkflowTemplateVersionDB(namespace, name, originalVersionAsString) + + assert.False(t, updated.IsLatest) +} + +// Test_getWorkflowTemplate_SuccessVersion tests cases for creating a workflow template version +func TestClient_CreateWorkflowTemplateVersion(t *testing.T) { + testClientCreateWorkflowTemplateVersionNew(t) + testClientCreateWorkflowTemplateVersionMarkOldNotLatest(t) +} + +// testGetWorkflowTemplateSuccess gets a workflow template with no error conditions encountered +func testClientGetWorkflowTemplateSuccess(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + workflowTemplate := &WorkflowTemplate{ + Name: "test", + Manifest: defaultWorkflowTemplate, + } + created, _ := c.CreateWorkflowTemplate(namespace, workflowTemplate) + + wt, err := c.GetWorkflowTemplate(namespace, created.UID, 0) + assert.NotNil(t, wt) + assert.Nil(t, err) +} + +// testGetWorkflowTemplateNotFound attempts to get a not-found workflow template +func testClientGetWorkflowTemplateNotFound(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + wt, err := c.GetWorkflowTemplate("onepanel", "uid-not-found", 0) + assert.Nil(t, wt) + + userErr, ok := err.(*util.UserError) + assert.True(t, ok) + + assert.Equal(t, codes.NotFound, userErr.Code) +} + +func TestClient_GetWorkflowTemplate(t *testing.T) { + testClientGetWorkflowTemplateSuccess(t) + testClientGetWorkflowTemplateNotFound(t) +} diff --git a/pkg/workflow_template_types.go b/pkg/workflow_template_types.go index 292a2f6..03e29bd 100644 --- a/pkg/workflow_template_types.go +++ b/pkg/workflow_template_types.go @@ -4,6 +4,7 @@ import ( "encoding/json" wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" "github.com/onepanelio/core/pkg/util/mapping" + uid2 "github.com/onepanelio/core/pkg/util/uid" "github.com/onepanelio/core/util/sql" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -13,6 +14,9 @@ import ( // WorkflowTemplate represents a Workflow Template backed by a database row // it stores information required to run an execution +// A Workflow template is uniquely identified by +// (namespace, uid, is_archived) +// (namespace, name, is_archived) -- because we create a uid from the name. type WorkflowTemplate struct { ID uint64 CreatedAt time.Time `db:"created_at"` @@ -35,6 +39,18 @@ type WorkflowTemplate struct { ResourceUID *string // see Resource field } +// GenerateUID generates a uid from the input name and sets it on the workflow template +func (wt *WorkflowTemplate) GenerateUID(name string) error { + result, err := uid2.GenerateUID(name, 30) + if err != nil { + return err + } + + wt.UID = result + + return nil +} + // GetManifestBytes returns the manifest as []byte func (wt *WorkflowTemplate) GetManifestBytes() []byte { return []byte(wt.Manifest) diff --git a/pkg/workspace.go b/pkg/workspace.go index e6fe3f5..a3a4678 100644 --- a/pkg/workspace.go +++ b/pkg/workspace.go @@ -10,9 +10,9 @@ import ( "github.com/onepanelio/core/pkg/util" "github.com/onepanelio/core/pkg/util/pagination" "github.com/onepanelio/core/pkg/util/ptr" - uid2 "github.com/onepanelio/core/pkg/util/uid" log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" + "strings" "time" ) @@ -33,16 +33,55 @@ func (c *Client) workspacesSelectBuilder(namespace string) sq.SelectBuilder { return sb } -func getWorkspaceParameterValue(parameters []Parameter, name string) *string { - for _, p := range parameters { - if p.Name == name { - return p.Value - } +// workspaceStatusToFieldMap takes a status and creates a map of the fields that should be updated +func workspaceStatusToFieldMap(status *WorkspaceStatus) sq.Eq { + fieldMap := sq.Eq{ + "phase": status.Phase, + "modified_at": time.Now().UTC(), + } + switch status.Phase { + case WorkspaceLaunching: + fieldMap["paused_at"] = pq.NullTime{} + fieldMap["started_at"] = time.Now().UTC() + break + case WorkspacePausing: + fieldMap["started_at"] = pq.NullTime{} + fieldMap["paused_at"] = time.Now().UTC() + break + case WorkspaceUpdating: + fieldMap["paused_at"] = pq.NullTime{} + fieldMap["updated_at"] = time.Now().UTC() + break + case WorkspaceTerminating: + fieldMap["started_at"] = pq.NullTime{} + fieldMap["paused_at"] = pq.NullTime{} + fieldMap["terminated_at"] = time.Now().UTC() + break } - return nil + return fieldMap } +// updateWorkspaceStatusBuilder creates an update builder that updates a workspace's status and related fields to match that status. +func updateWorkspaceStatusBuilder(namespace, uid string, status *WorkspaceStatus) sq.UpdateBuilder { + fieldMap := workspaceStatusToFieldMap(status) + + ub := sb.Update("workspaces"). + SetMap(fieldMap). + Where(sq.And{ + sq.Eq{ + "namespace": namespace, + "uid": uid, + }, sq.NotEq{ + "phase": WorkspaceTerminated, + }, + }) + + return ub +} + +// mergeWorkspaceParameters combines two parameter arrays. If a parameter in newParameters is not in +// the existing ones, it is added. If it is, it is ignored. func mergeWorkspaceParameters(existingParameters, newParameters []Parameter) (parameters []Parameter) { parameterMap := make(map[string]*string, 0) for _, p := range newParameters { @@ -74,10 +113,6 @@ func mergeWorkspaceParameters(existingParameters, newParameters []Parameter) (pa // sys-resource-action // sys-host func injectWorkspaceSystemParameters(namespace string, workspace *Workspace, workspaceAction, resourceAction string, config SystemConfig) (err error) { - workspace.UID, err = uid2.GenerateUID(workspace.Name, 30) - if err != nil { - return - } host := fmt.Sprintf("%v--%v.%v", workspace.UID, namespace, *config.Domain()) systemParameters := []Parameter{ { @@ -98,6 +133,10 @@ func injectWorkspaceSystemParameters(namespace string, workspace *Workspace, wor return } +// createWorkspace creates a workspace and related resources. +// The following are required on the workspace: +// WorkspaceTemplate.WorkflowTemplate.UID +// WorkspaceTemplate.WorkflowTemplate.Version func (c *Client) createWorkspace(namespace string, parameters []byte, workspace *Workspace) (*Workspace, error) { systemConfig, err := c.GetSystemConfig() if err != nil { @@ -155,7 +194,11 @@ func (c *Client) createWorkspace(namespace string, parameters []byte, workspace QueryRow(). Scan(&workspace.ID, &workspace.CreatedAt) if err != nil { - return nil, util.NewUserErrorWrap(err, "Workspace") + if strings.Contains(err.Error(), "invalid input syntax for type json") { + return nil, util.NewUserError(codes.InvalidArgument, err.Error()) + } + + return nil, util.NewUserError(codes.Unknown, err.Error()) } return workspace, nil @@ -168,6 +211,10 @@ func (c *Client) CreateWorkspace(namespace string, workspace *Workspace) (*Works return nil, err } + if err := workspace.GenerateUID(workspace.Name); err != nil { + return nil, err + } + parameters, err := json.Marshal(workspace.Parameters) if err != nil { return nil, err @@ -182,7 +229,7 @@ func (c *Client) CreateWorkspace(namespace string, workspace *Workspace) (*Works Value: ptr.String(workspace.UID), }) - sysHost := getWorkspaceParameterValue(workspace.Parameters, "sys-host") + sysHost := workspace.GetParameterValue("sys-host") if sysHost == nil { return nil, fmt.Errorf("sys-host parameter not found") } @@ -267,51 +314,31 @@ func (c *Client) GetWorkspace(namespace, uid string) (workspace *Workspace, err // UpdateWorkspaceStatus updates workspace status and times based on phase func (c *Client) UpdateWorkspaceStatus(namespace, uid string, status *WorkspaceStatus) (err error) { - fieldMap := sq.Eq{ - "phase": status.Phase, - "modified_at": time.Now().UTC(), - } - switch status.Phase { - case WorkspaceLaunching: - fieldMap["paused_at"] = pq.NullTime{} - fieldMap["started_at"] = time.Now().UTC() - break - case WorkspacePausing: - fieldMap["started_at"] = pq.NullTime{} - fieldMap["paused_at"] = time.Now().UTC() - break - case WorkspaceUpdating: - fieldMap["paused_at"] = pq.NullTime{} - fieldMap["updated_at"] = time.Now().UTC() - break - case WorkspaceTerminating: - fieldMap["started_at"] = pq.NullTime{} - fieldMap["paused_at"] = pq.NullTime{} - fieldMap["terminated_at"] = time.Now().UTC() - break - } - _, err = sb.Update("workspaces"). - SetMap(fieldMap). - Where(sq.And{ - sq.Eq{ - "namespace": namespace, - "uid": uid, - }, sq.NotEq{ - "phase": WorkspaceTerminated, - }, - }). - RunWith(c.DB).Exec() + result, err := updateWorkspaceStatusBuilder(namespace, uid, status). + RunWith(c.DB). + Exec() if err != nil { + return err + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return err + } + + if rowsAffected == 0 { return util.NewUserError(codes.NotFound, "Workspace not found.") } return } -// ListWorkspacesByTemplateID will return all the workspaces for a given workspace template id. +// ListWorkspacesByTemplateID will return all the workspaces for a given workspace template id that are not terminated. // Sourced from database. +// Includes labels. func (c *Client) ListWorkspacesByTemplateID(namespace string, templateID uint64) (workspaces []*Workspace, err error) { sb := sb.Select(getWorkspaceColumns("w")...). + Columns(getWorkspaceStatusColumns("w", "status")...). From("workspaces w"). Where(sq.And{ sq.Eq{ @@ -391,6 +418,7 @@ func (c *Client) CountWorkspaces(namespace string) (count int, err error) { return } +// updateWorkspace updates the workspace to the indicated status func (c *Client) updateWorkspace(namespace, uid, workspaceAction, resourceAction string, status *WorkspaceStatus, parameters ...Parameter) (err error) { workspace, err := c.GetWorkspace(namespace, uid) if err != nil { @@ -444,32 +472,15 @@ func (c *Client) updateWorkspace(namespace, uid, workspaceAction, resourceAction return } - if err = c.UpdateWorkspaceStatus(namespace, uid, status); err != nil { - return + sb := updateWorkspaceStatusBuilder(namespace, uid, status) + + // Update parameters if they are passed in + if len(parameters) != 0 { + sb.Set("parameters", parametersJSON) } - // Update parameters if they are passed - if len(parameters) == 0 { - return - } - - _, err = sb.Update("workspaces"). - SetMap(sq.Eq{ - "parameters": parametersJSON, - }). - Where(sq.And{ - sq.Eq{ - "namespace": namespace, - "uid": uid, - }, sq.NotEq{ - "phase": WorkspaceTerminated, - }, - }). - RunWith(c.DB). + _, err = sb.RunWith(c.DB). Exec() - if err != nil { - return util.NewUserError(codes.NotFound, "Workspace not found.") - } return } @@ -492,6 +503,6 @@ func (c *Client) DeleteWorkspace(namespace, uid string) (err error) { // ArchiveWorkspace archives by setting the workspace to delete or terminate. // Kicks off DB archiving and k8s cleaning. -func (c *Client) ArchiveWorkspace(namespace, uid string) (err error) { - return c.updateWorkspace(namespace, uid, "delete", "delete", &WorkspaceStatus{Phase: WorkspaceTerminating}) +func (c *Client) ArchiveWorkspace(namespace, uid string, parameters ...Parameter) (err error) { + return c.updateWorkspace(namespace, uid, "delete", "delete", &WorkspaceStatus{Phase: WorkspaceTerminating}, parameters...) } diff --git a/pkg/workspace_template.go b/pkg/workspace_template.go index b5f063f..404a086 100644 --- a/pkg/workspace_template.go +++ b/pkg/workspace_template.go @@ -12,7 +12,6 @@ import ( "github.com/onepanelio/core/pkg/util/env" "github.com/onepanelio/core/pkg/util/pagination" "github.com/onepanelio/core/pkg/util/ptr" - uid2 "github.com/onepanelio/core/pkg/util/uid" log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" networking "istio.io/api/networking/v1alpha3" @@ -118,82 +117,6 @@ func generateRuntimeParameters(config SystemConfig) (parameters []Parameter, err return } -func generateStaticParameters() (parameters []Parameter, err error) { - parameters = make([]Parameter, 0) - - // Resource action parameter - parameters = append(parameters, Parameter{ - Name: "sys-name", - Type: "input.text", - Value: ptr.String("name"), - DisplayName: ptr.String("Workspace name"), - Hint: ptr.String("Must be between 3-30 characters, contain only alphanumeric or `-` characters"), - Required: true, - }) - - // TODO: These can be removed when lint validation of workflows work - // Resource action parameter - parameters = append(parameters, Parameter{ - Name: "sys-resource-action", - Value: ptr.String("apply"), - Type: "input.hidden", - }) - // Workspace action - parameters = append(parameters, Parameter{ - Name: "sys-workspace-action", - Value: ptr.String("create"), - Type: "input.hidden", - }) - - // UID placeholder - parameters = append(parameters, Parameter{ - Name: "sys-uid", - Value: ptr.String("uid"), - Type: "input.hidden", - }) - - return -} - -func generateVolumeParameters(spec *WorkspaceSpec) (parameters []Parameter, err error) { - if spec == nil { - return nil, fmt.Errorf("workspaceSpec is nil") - } - - parameters = make([]Parameter, 0) - - // Map all the volumeClaimTemplates that have storage set - volumeStorageQuantityIsSet := make(map[string]bool) - for _, v := range spec.VolumeClaimTemplates { - if v.Spec.Resources.Requests != nil { - volumeStorageQuantityIsSet[v.ObjectMeta.Name] = true - } - } - // Volume size parameters - volumeClaimsMapped := make(map[string]bool) - for _, c := range spec.Containers { - for _, v := range c.VolumeMounts { - // Skip if already mapped or storage size is set - if volumeClaimsMapped[v.Name] || volumeStorageQuantityIsSet[v.Name] { - continue - } - - parameters = append(parameters, Parameter{ - Name: fmt.Sprintf("sys-%v-volume-size", v.Name), - Type: "input.number", - Value: ptr.String("20480"), - DisplayName: ptr.String(fmt.Sprintf("Disk size for \"%v\"", v.Name)), - Hint: ptr.String(fmt.Sprintf("Disk size in MB for volume mounted at `%v`", v.MountPath)), - Required: true, - }) - - volumeClaimsMapped[v.Name] = true - } - } - - return -} - func generateArguments(spec *WorkspaceSpec, config SystemConfig) (err error) { systemParameters := make([]Parameter, 0) // Resource action parameter @@ -726,15 +649,14 @@ metadata: } func (c *Client) createWorkspaceTemplate(namespace string, workspaceTemplate *WorkspaceTemplate) (*WorkspaceTemplate, error) { - uid, err := uid2.GenerateUID(workspaceTemplate.Name, 30) + err := workspaceTemplate.GenerateUID(workspaceTemplate.Name) if err != nil { return nil, err } - workspaceTemplate.UID = uid workspaceTemplate.WorkflowTemplate.IsSystem = true workspaceTemplate.WorkflowTemplate.Resource = ptr.String(TypeWorkspaceTemplate) - workspaceTemplate.WorkflowTemplate.ResourceUID = ptr.String(uid) + workspaceTemplate.WorkflowTemplate.ResourceUID = &workspaceTemplate.UID // validate workflow template if err := c.validateWorkflowTemplate(namespace, workspaceTemplate.WorkflowTemplate); err != nil { @@ -761,7 +683,7 @@ func (c *Client) createWorkspaceTemplate(namespace string, workspaceTemplate *Wo defer tx.Rollback() err = sb.Insert("workspace_templates"). SetMap(sq.Eq{ - "uid": uid, + "uid": workspaceTemplate.UID, "name": workspaceTemplate.Name, "namespace": namespace, "workflow_template_id": workspaceTemplate.WorkflowTemplate.ID, @@ -1049,26 +971,23 @@ func (c *Client) UpdateWorkspaceTemplate(namespace string, workspaceTemplate *Wo return workspaceTemplate, nil } +// ListWorkspaceTemplates returns a list of workspace templates that are not archived, sorted by most recent created first func (c *Client) ListWorkspaceTemplates(namespace string, paginator *pagination.PaginationRequest) (workspaceTemplates []*WorkspaceTemplate, err error) { sb := c.workspaceTemplatesSelectBuilder(namespace). Where(sq.Eq{ "wt.is_archived": false, }). OrderBy("wt.created_at DESC") + sb = *paginator.ApplyToSelect(&sb) - query, args, err := sb.ToSql() - if err != nil { - return nil, err - } - - if err := c.DB.Select(&workspaceTemplates, query, args...); err != nil { - return nil, err - } + err = c.DB.Selectx(&workspaceTemplates, sb) return } +// ListWorkspaceTemplateVersions returns an array of WorkspaceTemplates with the version information loaded. Latest id is first. +// Labels are also loaded. func (c *Client) ListWorkspaceTemplateVersions(namespace, uid string) (workspaceTemplates []*WorkspaceTemplate, err error) { sb := c.workspaceTemplateVersionsSelectBuilder(namespace, uid). Options("DISTINCT ON (wtv.version) wtv.version,"). @@ -1077,11 +996,8 @@ func (c *Client) ListWorkspaceTemplateVersions(namespace, uid string) (workspace "wft.is_archived": false, }). OrderBy("wtv.version DESC") - query, args, err := sb.ToSql() - if err != nil { - return - } - if err = c.DB.Select(&workspaceTemplates, query, args...); err != nil { + + if err = c.DB.Selectx(&workspaceTemplates, sb); err != nil { return } @@ -1099,6 +1015,7 @@ func (c *Client) ListWorkspaceTemplateVersions(namespace, uid string) (workspace return } +// CountWorkspaceTemplates returns the total number of non-archived workspace templates for the input namespace func (c *Client) CountWorkspaceTemplates(namespace string) (count int, err error) { err = sb.Select("count(*)"). From("workspace_templates wt"). @@ -1189,6 +1106,9 @@ func (c *Client) ArchiveWorkspaceTemplate(namespace string, uid string) (archive }).Error("Get Workspace Template failed.") return false, util.NewUserError(codes.Unknown, "Unable to archive workspace template.") } + if wsTemp == nil { + return false, fmt.Errorf("not found") + } wsList, err := c.ListWorkspacesByTemplateID(namespace, wsTemp.WorkspaceTemplateVersionID) if err != nil { diff --git a/pkg/workspace_template_test.go b/pkg/workspace_template_test.go index fe993b3..80afef6 100644 --- a/pkg/workspace_template_test.go +++ b/pkg/workspace_template_test.go @@ -52,9 +52,78 @@ routes: workspaceTemplate = WorkspaceTemplate{ Manifest: workspaceSpecManifest, } + + jupyterLabWorkspaceManifest = `# Docker containers that are part of the Workspace +containers: +- name: jupyterlab-tensorflow + image: jupyter/tensorflow-notebook + command: [start.sh, jupyter] + workingDir: /data + env: + - name: tornado + value: "{ 'headers': { 'Content-Security-Policy': \"frame-ancestors * 'self'\" } }" + - name: GRANT_SUDO + value: 1 + - name: CHOWN_EXTRA + value: '/data' + - name: CHOWN_EXTRA_OPTS + value: '-R' + securityContext: + runAsUser: 0 + allowPrivilegeEscalation: false + args: + - lab + - --LabApp.token='' + - --LabApp.allow_remote_access=True + - --LabApp.allow_origin="*" + - --LabApp.disable_check_xsrf=True + - --LabApp.trust_xheaders=True + - --LabApp.tornado_settings=$(tornado) + - --NotebookApp.notebook_dir='/data' + ports: + - containerPort: 8888 + name: jupyterlab + # Volumes to be mounted in this container + # Onepanel will automatically create these volumes and mount them to the container + volumeMounts: + - name: data + mountPath: /data +# Ports that need to be exposed +ports: +- name: jupyterlab + port: 80 + protocol: TCP + targetPort: 8888 +# Routes that will map to ports +routes: +- match: + - uri: + prefix: / + route: + - destination: + port: + number: 80 +# DAG Workflow to be executed once a Workspace action completes +# postExecutionWorkflow: +# entrypoint: main +# templates: +# - name: main +# dag: +# tasks: +# - name: slack-notify +# template: slack-notify +# - name: slack-notify +# container: +# image: technosophos/slack-notify +# args: +# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify +# command: +# - sh +# - -c +` ) -func TestParseWorkspaceSpec(t *testing.T) { +func Test_ParseWorkspaceSpec(t *testing.T) { workspaceSpec, err := parseWorkspaceSpec(workspaceSpecManifest) assert.Nil(t, err) assert.NotEmpty(t, workspaceSpec) @@ -65,3 +134,151 @@ func TestParseWorkspaceSpec(t *testing.T) { assert.Equal(t, workspaceSpec.Containers[0].Ports[0].ContainerPort, int32(80)) assert.Equal(t, workspaceSpec.Containers[1].Ports[0].ContainerPort, int32(443)) } + +// testClientCreateWorkspaceTemplateNew creates a new workspace template +func testClientCreateWorkspaceTemplateNew(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + wt := &WorkspaceTemplate{ + Name: "test", + Manifest: jupyterLabWorkspaceManifest, + } + + _, err := c.CreateWorkspaceTemplate(namespace, wt) + + assert.Nil(t, err) +} + +// testClientCreateWorkspaceTemplateDuplicateName attempts to create a workspace template for a name that already exists +// this should error +func testClientCreateWorkspaceTemplateDuplicateName(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + wt := &WorkspaceTemplate{ + Name: "test", + Manifest: jupyterLabWorkspaceManifest, + } + + _, err := c.CreateWorkspaceTemplate(namespace, wt) + _, err = c.CreateWorkspaceTemplate(namespace, wt) + + assert.NotNil(t, err) +} + +// testClientCreateWorkspaceTemplateArchivedName attempts to create a workspace template for a name that has been archived +// this should work +func testClientCreateWorkspaceTemplateArchivedName(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + wt := &WorkspaceTemplate{ + Name: "test", + Manifest: jupyterLabWorkspaceManifest, + } + + wtCreated, err := c.CreateWorkspaceTemplate(namespace, wt) + _, err = c.ArchiveWorkspaceTemplate(namespace, wtCreated.UID) + _, err = c.CreateWorkspaceTemplate(namespace, wt) + + assert.Nil(t, err) +} + +// TestClient_CreateWorkspaceTemplate tests creating a workspace template +func TestClient_CreateWorkspaceTemplate(t *testing.T) { + testClientCreateWorkspaceTemplateNew(t) + testClientCreateWorkspaceTemplateDuplicateName(t) + testClientCreateWorkspaceTemplateArchivedName(t) +} + +func testClientArchiveWorkspaceTemplateSuccess(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + wt := &WorkspaceTemplate{ + Name: "test", + Manifest: jupyterLabWorkspaceManifest, + } + + testTemplate, _ := c.CreateWorkspaceTemplate(namespace, wt) + archived, err := c.ArchiveWorkspaceTemplate(namespace, testTemplate.UID) + + assert.Nil(t, err) + assert.True(t, archived) +} + +// testClientArchiveWorkspaceTemplateNotFound tests the case where you try to archive a non-existing workspace template +func testClientArchiveWorkspaceTemplateNotFound(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + archived, err := c.ArchiveWorkspaceTemplate(namespace, "not-found") + + assert.NotNil(t, err) + assert.False(t, archived) +} + +// TestClient_ArchiveWorkspaceTemplate tests archiving a workspace template +func TestClient_ArchiveWorkspaceTemplate(t *testing.T) { + testClientArchiveWorkspaceTemplateSuccess(t) + testClientArchiveWorkspaceTemplateNotFound(t) + + // TODO we need more tests here to make sure the related resources are cleaned up, including workspaces and workflow templates +} + +// testClientListWorkspaceTemplatesEmpty tests listing workspace templates when there are none +func testClientListWorkspaceTemplatesEmpty(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + templates, err := c.ListWorkspaceTemplates("onepanel", nil) + + assert.Nil(t, err) + assert.Empty(t, templates) +} + +// testClientListWorkspaceTemplatesNotEmpty tests listing workspaces when there are records that are +// archived and not. It should only list the non-archived ones +func testClientListWorkspaceTemplatesNotEmpty(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + wt := &WorkspaceTemplate{ + Name: "test", + Manifest: jupyterLabWorkspaceManifest, + } + + testTemplate, _ := c.CreateWorkspaceTemplate(namespace, wt) + c.ArchiveWorkspaceTemplate(namespace, testTemplate.UID) + + wt2 := &WorkspaceTemplate{ + Name: "test2", + Manifest: jupyterLabWorkspaceManifest, + } + + wt2, _ = c.CreateWorkspaceTemplate(namespace, wt2) + + templates, err := c.ListWorkspaceTemplates(namespace, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(templates)) + assert.Equal(t, wt2.UID, templates[0].UID) +} + +// TestClient_ListWorkspaceTemplates tests listing workspace templates +func TestClient_ListWorkspaceTemplates(t *testing.T) { + testClientListWorkspaceTemplatesEmpty(t) + testClientListWorkspaceTemplatesNotEmpty(t) +} diff --git a/pkg/workspace_template_types.go b/pkg/workspace_template_types.go index 86deaa7..e641561 100644 --- a/pkg/workspace_template_types.go +++ b/pkg/workspace_template_types.go @@ -3,6 +3,7 @@ package v1 import ( "fmt" wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" + uid2 "github.com/onepanelio/core/pkg/util/uid" "github.com/onepanelio/core/util/sql" "sigs.k8s.io/yaml" "time" @@ -25,6 +26,18 @@ type WorkspaceTemplate struct { WorkflowTemplateID uint64 `db:"workflow_template_id"` } +// GenerateUID generates a uid from the input name and sets it on the workflow template +func (wt *WorkspaceTemplate) GenerateUID(name string) error { + result, err := uid2.GenerateUID(name, 30) + if err != nil { + return err + } + + wt.UID = result + + return nil +} + // InjectRuntimeParameters will inject all runtime variables into the WorkflowTemplate's manifest. func (wt *WorkspaceTemplate) InjectRuntimeParameters(config SystemConfig) error { if wt.WorkflowTemplate == nil { diff --git a/pkg/workspace_test.go b/pkg/workspace_test.go new file mode 100644 index 0000000..478cbac --- /dev/null +++ b/pkg/workspace_test.go @@ -0,0 +1,314 @@ +package v1 + +import ( + "github.com/lib/pq" + "github.com/onepanelio/core/pkg/util" + "github.com/onepanelio/core/pkg/util/ptr" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "testing" + "time" +) + +func testWorkspaceStatusToFieldMapLaunching(t *testing.T) { + fm := workspaceStatusToFieldMap(&WorkspaceStatus{Phase: WorkspaceLaunching}) + + assert.Equal(t, fm["phase"], WorkspaceLaunching) + assert.Equal(t, fm["paused_at"], pq.NullTime{}) + + started := fm["started_at"].(time.Time) + + assert.True(t, started.After(time.Time{})) +} + +func testWorkspaceStatusToFieldMapPausing(t *testing.T) { + fm := workspaceStatusToFieldMap(&WorkspaceStatus{Phase: WorkspacePausing}) + + assert.Equal(t, fm["phase"], WorkspacePausing) + assert.Equal(t, fm["started_at"], pq.NullTime{}) + + paused := fm["paused_at"].(time.Time) + + assert.True(t, paused.After(time.Time{})) +} + +func testWorkspaceStatusToFieldMapUpdating(t *testing.T) { + fm := workspaceStatusToFieldMap(&WorkspaceStatus{Phase: WorkspaceUpdating}) + + assert.Equal(t, fm["phase"], WorkspaceUpdating) + assert.Equal(t, fm["paused_at"], pq.NullTime{}) + + updated := fm["updated_at"].(time.Time) + + assert.True(t, updated.After(time.Time{})) +} + +func testWorkspaceStatusToFieldMapTerminating(t *testing.T) { + fm := workspaceStatusToFieldMap(&WorkspaceStatus{Phase: WorkspaceTerminating}) + + assert.Equal(t, fm["phase"], WorkspaceTerminating) + assert.Equal(t, fm["paused_at"], pq.NullTime{}) + assert.Equal(t, fm["started_at"], pq.NullTime{}) + + terminated := fm["terminated_at"].(time.Time) + + assert.True(t, terminated.After(time.Time{})) +} + +func Test_WorkspaceStatusToFieldMap(t *testing.T) { + testWorkspaceStatusToFieldMapLaunching(t) + testWorkspaceStatusToFieldMapPausing(t) + testWorkspaceStatusToFieldMapUpdating(t) + testWorkspaceStatusToFieldMapTerminating(t) +} + +// testClientPrivateCreateWorkspaceNoWorkflowTemplate makes sure we get an error when there is no workflow template for the workspace +func testClientPrivateCreateWorkspaceNoWorkflowTemplate(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + workspace := &Workspace{ + Name: "test", + WorkspaceTemplate: &WorkspaceTemplate{ + WorkflowTemplate: &WorkflowTemplate{ + UID: "not-exist", + Version: 1, + }, + }, + } + workspace.GenerateUID("test") + + _, err := c.createWorkspace(namespace, []byte(""), workspace) + + userErr, ok := err.(*util.UserError) + assert.True(t, ok) + assert.Equal(t, userErr.Code, codes.NotFound) +} + +// testClientPrivateCreateWorkspaceSuccess tests creating a workspace successfully +func testClientPrivateCreateWorkspaceSuccess(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + workspaceTemplate := &WorkspaceTemplate{ + Name: "test", + Manifest: jupyterLabWorkspaceManifest, + } + + workspaceTemplate, _ = c.CreateWorkspaceTemplate(namespace, workspaceTemplate) + + workspace := &Workspace{ + Name: "test2", + WorkspaceTemplate: workspaceTemplate, + Parameters: []Parameter{ + { + Name: "workflow-execution-name", + Value: ptr.String("test2"), + }, + }, + } + workspace.GenerateUID("test") + + _, err := c.createWorkspace(namespace, []byte("{}"), workspace) + + assert.Nil(t, err) +} + +func TestClient_createWorkspace(t *testing.T) { + testClientPrivateCreateWorkspaceNoWorkflowTemplate(t) + testClientPrivateCreateWorkspaceSuccess(t) +} + +func testClientCreateWorkspaceSuccess(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + wt := &WorkspaceTemplate{ + Name: "test", + Manifest: jupyterLabWorkspaceManifest, + } + + testTemplate, _ := c.CreateWorkspaceTemplate(namespace, wt) + + workspace := &Workspace{ + Name: "test", + WorkspaceTemplate: &WorkspaceTemplate{ + UID: testTemplate.UID, + Version: testTemplate.Version, + }, + Parameters: []Parameter{ + { + Name: "workflow-execution-name", + Value: ptr.String("test2"), + }, + }, + } + + createdWorkspace, err := c.CreateWorkspace(namespace, workspace) + + assert.Nil(t, err) + assert.NotNil(t, createdWorkspace) +} + +func TestClient_CreateWorkspace(t *testing.T) { + testClientCreateWorkspaceSuccess(t) +} + +func TestClient_ArchiveWorkspace(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + workspaceTemplate := &WorkspaceTemplate{ + Name: "test", + Manifest: jupyterLabWorkspaceManifest, + } + + workspaceTemplate, _ = c.CreateWorkspaceTemplate(namespace, workspaceTemplate) + + workspace := &Workspace{ + Name: "test2", + WorkspaceTemplate: workspaceTemplate, + Parameters: []Parameter{ + { + Name: "workflow-execution-name", + Value: ptr.String("test2"), + }, + }, + } + workspace.GenerateUID("test") + + createdWorkspace, _ := c.createWorkspace(namespace, []byte("[]"), workspace) + + params := []Parameter{ + { + Name: "workflow-execution-name", + Value: ptr.String("test3"), + }, + } + err := c.ArchiveWorkspace(namespace, createdWorkspace.UID, params...) + + assert.Nil(t, err) +} + +// TestClient_ListWorkspacesByTemplateID tests listing workspaces by the template id +func TestClient_ListWorkspacesByTemplateID(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + wt := &WorkspaceTemplate{ + Name: "test", + Manifest: jupyterLabWorkspaceManifest, + } + + testTemplate, _ := c.CreateWorkspaceTemplate(namespace, wt) + workspace := &Workspace{ + Name: "test", + WorkspaceTemplate: testTemplate, + Parameters: []Parameter{ + { + Name: "workflow-execution-name", + Value: ptr.String("test"), + }, + }, + } + workspace.GenerateUID("test") + + c.createWorkspace(namespace, []byte("[]"), workspace) + + wt2 := &WorkspaceTemplate{ + Name: "test2", + Manifest: jupyterLabWorkspaceManifest, + } + + testTemplate2, _ := c.CreateWorkspaceTemplate(namespace, wt2) + workspace2 := &Workspace{ + Name: "test2", + WorkspaceTemplate: testTemplate2, + Parameters: []Parameter{ + { + Name: "workflow-execution-name", + Value: ptr.String("test2"), + }, + }, + } + workspace2.GenerateUID("test2") + + c.createWorkspace(namespace, []byte("[]"), workspace2) + + workspaces, err := c.ListWorkspacesByTemplateID(namespace, testTemplate.ID) + assert.Nil(t, err) + assert.Equal(t, 1, len(workspaces)) + + params := []Parameter{ + { + Name: "workflow-execution-name", + Value: ptr.String("test3"), + }, + } + c.ArchiveWorkspace(namespace, testTemplate.UID, params...) + + workspaces, err = c.ListWorkspacesByTemplateID(namespace, testTemplate.ID) + assert.Nil(t, err) + assert.True(t, workspaces[0].Status.Phase == WorkspaceTerminating) +} + +func testUpdateWorkspaceStatusSuccess(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + namespace := "onepanel" + + workspace := &Workspace{ + Name: "test", + Parameters: []Parameter{ + { + Name: "workflow-execution-name", + Value: ptr.String("test"), + }, + }, + WorkspaceTemplate: &WorkspaceTemplate{ + Name: "test", + Manifest: jupyterLabWorkspaceManifest, + WorkflowTemplate: &WorkflowTemplate{ + UID: "test", + Version: 1, + }, + }, + } + workspace.GenerateUID("test") + + c.CreateWorkspaceTemplate(namespace, workspace.WorkspaceTemplate) + ws, _ := c.createWorkspace(namespace, []byte("[]"), workspace) + + err := c.UpdateWorkspaceStatus(namespace, ws.UID, &WorkspaceStatus{Phase: WorkspacePausing}) + + assert.Nil(t, err) +} + +func testUpdateWorkspaceStatusNotFound(t *testing.T) { + c := DefaultTestClient() + clearDatabase(t) + + err := c.UpdateWorkspaceStatus("not-found", "random", &WorkspaceStatus{Phase: WorkspacePausing}) + assert.NotNil(t, err) + + userErr, ok := err.(*util.UserError) + assert.True(t, ok) + + assert.Equal(t, userErr.Code, codes.NotFound) +} + +func Test_UpdateWorkspaceStatus(t *testing.T) { + testUpdateWorkspaceStatusSuccess(t) + testUpdateWorkspaceStatusNotFound(t) +} diff --git a/pkg/workspace_types.go b/pkg/workspace_types.go index bd95f09..1753b2a 100644 --- a/pkg/workspace_types.go +++ b/pkg/workspace_types.go @@ -3,6 +3,7 @@ package v1 import ( "fmt" wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" + uid2 "github.com/onepanelio/core/pkg/util/uid" "github.com/onepanelio/core/util/sql" networking "istio.io/api/networking/v1alpha3" corev1 "k8s.io/api/core/v1" @@ -63,6 +64,29 @@ func (w *Workspace) GetURL(protocol, domain string) string { return fmt.Sprintf("%v%v--%v.%v", protocol, w.UID, w.Namespace, domain) } +// GetParameterValue returns the value of the parameter with the given name, or nil if there is no such parameter +func (w *Workspace) GetParameterValue(name string) *string { + for _, p := range w.Parameters { + if p.Name == name { + return p.Value + } + } + + return nil +} + +// GenerateUID generates a uid from the input name and sets it on the workspace +func (w *Workspace) GenerateUID(name string) error { + result, err := uid2.GenerateUID(name, 30) + if err != nil { + return err + } + + w.UID = result + + return nil +} + // getWorkspaceColumns returns all of the columns for workspace modified by alias, destination. // see formatColumnSelect func getWorkspaceColumns(aliasAndDestination ...string) []string { diff --git a/pkg/workspace_types_test.go b/pkg/workspace_types_test.go index f4bb31f..e30e835 100644 --- a/pkg/workspace_types_test.go +++ b/pkg/workspace_types_test.go @@ -28,7 +28,7 @@ func assertWorkspaceNameValid(t *testing.T, name string) { assert.True(t, valid) } -func TestWorkspaceNameValidation_RegexValid(t *testing.T) { +func Test_WorkspaceNameValidation_RegexValid(t *testing.T) { assertWorkspaceNameInvalid(t, "600s") assertWorkspaceNameValid(t, "test-5") diff --git a/server/workflow_template_server.go b/server/workflow_template_server.go index e61da3d..a4be0f6 100644 --- a/server/workflow_template_server.go +++ b/server/workflow_template_server.go @@ -105,27 +105,6 @@ func (s *WorkflowTemplateServer) CreateWorkflowTemplateVersion(ctx context.Conte return req.WorkflowTemplate, nil } -func (s *WorkflowTemplateServer) UpdateWorkflowTemplateVersion(ctx context.Context, req *api.UpdateWorkflowTemplateVersionRequest) (*api.WorkflowTemplate, error) { - client := getClient(ctx) - allowed, err := auth.IsAuthorized(client, req.Namespace, "update", "argoproj.io", "workflowtemplates", req.WorkflowTemplate.Name) - if err != nil || !allowed { - return nil, err - } - - workflowTemplate := &v1.WorkflowTemplate{ - UID: req.WorkflowTemplate.Uid, - Name: req.WorkflowTemplate.Name, - Manifest: req.WorkflowTemplate.Manifest, - Version: req.WorkflowTemplate.Version, - } - - req.WorkflowTemplate.Uid = workflowTemplate.UID - req.WorkflowTemplate.Name = workflowTemplate.Name - req.WorkflowTemplate.Version = workflowTemplate.Version - - return req.WorkflowTemplate, nil -} - func (s *WorkflowTemplateServer) GetWorkflowTemplate(ctx context.Context, req *api.GetWorkflowTemplateRequest) (*api.WorkflowTemplate, error) { client := getClient(ctx) allowed, err := auth.IsAuthorized(client, req.Namespace, "get", "argoproj.io", "workflowtemplates", "")