[Serving] Add a simple Python serving (#962)

* init simple serving

* simple serving is working

* ppyoloe demo

* Update README_CN.md

* update readme

* complete vision result to json

python/fastdeploy/serving/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+
+from .server import SimpleServer

python/fastdeploy/serving/handler/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from .base_handler import BaseModelHandler
+from .vision_model_handler import VisionModelHandler

python/fastdeploy/serving/handler/base_handler.py

@@ -0,0 +1,28 @@
+# coding:utf-8
+# Copyright (c) 2022  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+from abc import ABCMeta, abstractmethod
+
+
+class BaseModelHandler(metaclass=ABCMeta):
+    def __init__(self):
+        super().__init__()
+
+    @classmethod
+    @abstractmethod
+    def process(cls, predictor, data, parameters):
+        pass
+

python/fastdeploy/serving/handler/vision_model_handler.py

@@ -0,0 +1,30 @@
+# coding:utf-8
+# Copyright (c) 2022  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .base_handler import BaseModelHandler
+from ..utils import base64_to_cv2
+from ...vision.utils import fd_result_to_json
+
+
+class VisionModelHandler(BaseModelHandler):
+    def __init__(self):
+        super().__init__()
+
+    @classmethod
+    def process(cls, predictor, data, parameters):
+        # TODO: support batch predict
+        im = base64_to_cv2(data['image'])
+        result = predictor.predict(im)
+        r_str = fd_result_to_json(result)
+        return r_str

python/fastdeploy/serving/model_manager.py

@@ -0,0 +1,57 @@
+# coding:utf-8
+# copyright (c) 2022  paddlepaddle authors. all rights reserved.
+#
+# licensed under the apache license, version 2.0 (the "license"
+# you may not use this file except in compliance with the license.
+# you may obtain a copy of the license at
+#
+#     http://www.apache.org/licenses/license-2.0
+#
+# unless required by applicable law or agreed to in writing, software
+# distributed under the license is distributed on an "as is" basis,
+# without warranties or conditions of any kind, either express or implied.
+# see the license for the specific language governing permissions and
+# limitations under the license.
+
+import os
+import time
+import json
+import logging
+import threading
+# from .predictor import Predictor
+from .handler import BaseModelHandler
+from .utils import lock_predictor
+
+
+class ModelManager:
+    def __init__(self, model_handler, predictor):
+        self._model_handler = model_handler
+        self._predictors = []
+        self._predictor_locks = []
+        self._register(predictor)
+
+    def _register(self, predictor):
+        # Get the model handler
+        if not issubclass(self._model_handler, BaseModelHandler):
+            raise TypeError(
+                "The model_handler must be subclass of BaseModelHandler, please check the type."
+            )
+
+        # TODO: Create multiple predictors to run on different GPUs or different CPU threads
+        self._predictors.append(predictor)
+        self._predictor_locks.append(threading.Lock())
+
+    def _get_predict_id(self):
+        t = time.time()
+        t = int(round(t * 1000))
+        predictor_id = t % len(self._predictors)
+        logging.info("The predictor id: {} is selected by running the model.".
+                     format(predictor_id))
+        return predictor_id
+
+    def predict(self, data, parameters):
+        predictor_id = self._get_predict_id()
+        with lock_predictor(self._predictor_locks[predictor_id]):
+            model_output = self._model_handler.process(
+                self._predictors[predictor_id], data, parameters)
+            return model_output

python/fastdeploy/serving/router/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from .base_router import BaseRouterManager
+from .http_router import HttpRouterManager

python/fastdeploy/serving/router/base_router.py

@@ -0,0 +1,28 @@
+# coding:utf-8
+# Copyright (c) 2022  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+
+
+class BaseRouterManager(abc.ABC):
+    _app = None
+
+    def __init__(self, app):
+        super().__init__()
+        self._app = app
+
+    @abc.abstractmethod
+    def register_models_router(self):
+        return NotImplemented

python/fastdeploy/serving/router/http_router.py

@@ -0,0 +1,80 @@
+# coding:utf-8
+# Copyright (c) 2022  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import hashlib
+import typing
+import logging
+from typing import Optional
+
+from fastapi import APIRouter, Request, HTTPException
+from pydantic import BaseModel, Extra, create_model
+
+from .base_router import BaseRouterManager
+
+
+class ResponseBase(BaseModel):
+    text: Optional[str] = None
+
+
+class RequestBase(BaseModel, extra=Extra.forbid):
+    parameters: Optional[dict] = {}
+
+
+class HttpRouterManager(BaseRouterManager):
+    def register_models_router(self, task_name):
+
+        # Url path to register the model
+        paths = [f"/{task_name}"]
+        for path in paths:
+            logging.info("FastDeploy Model request [path]={} is genereated.".
+                         format(path))
+
+        # Unique name to create the pydantic model
+        unique_name = hashlib.md5(task_name.encode()).hexdigest()
+
+        # Create request model
+        req_model = create_model(
+            "RequestModel" + unique_name,
+            data=(typing.Any, ...),
+            __base__=RequestBase, )
+
+        # Create response model
+        resp_model = create_model(
+            "ResponseModel" + unique_name,
+            result=(typing.Any, ...),
+            __base__=ResponseBase, )
+
+        # Template predict endpoint function to dynamically serve different models
+        def predict(request: Request, inference_request: req_model):
+            try:
+                result = self._app._model_manager.predict(
+                    inference_request.data, inference_request.parameters)
+            except Exception as e:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Error occurred while running predict: {str(e)}")
+            return {"result": result}
+
+        # Register the route and add to the app
+        router = APIRouter()
+        for path in paths:
+            router.add_api_route(
+                path,
+                predict,
+                methods=["post"],
+                summary=f"{task_name.title()}",
+                response_model=resp_model,
+                response_model_exclude_unset=True,
+                response_model_exclude_none=True, )
+        self._app.include_router(router)

python/fastdeploy/serving/server.py

@@ -0,0 +1,46 @@
+# coding:utf-8
+# Copyright (c) 2022  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from fastapi import FastAPI
+from .router import HttpRouterManager
+from .model_manager import ModelManager
+
+
+class SimpleServer(FastAPI):
+    def __init__(self, **kwargs):
+        """
+        Initial function for the FastDeploy SimpleServer.
+        """
+        super().__init__(**kwargs)
+        self._router_manager = HttpRouterManager(self)
+        self._model_manager = None
+        self._service_name = "FastDeploy SimpleServer"
+        self._service_type = None
+
+    def register(self, task_name, model_handler, predictor):
+        """
+        The register function for the SimpleServer, the main register argrument as follows:
+
+        Args:
+            task_name(str): API URL path.
+            model_handler: To process request data, run predictor,
+                and can also add your custom post processing on top of the predictor result
+            predictor: To run model predict
+        """
+        self._server_type = "models"
+        model_manager = ModelManager(model_handler, predictor)
+        self._model_manager = model_manager
+        # Register model server router
+        self._router_manager.register_models_router(task_name)

python/fastdeploy/serving/utils.py

@@ -0,0 +1,40 @@
+# coding:utf-8
+# Copyright (c) 2022  PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import contextlib
+import base64
+import numpy as np
+import cv2
+
+
+@contextlib.contextmanager
+def lock_predictor(lock):
+    lock.acquire()
+    try:
+        yield
+    finally:
+        lock.release()
+
+
+def cv2_to_base64(image):
+    data = cv2.imencode('.jpg', image)[1]
+    return base64.b64encode(data.tobytes()).decode('utf8')
+
+
+def base64_to_cv2(b64str):
+    data = base64.b64decode(b64str.encode('utf8'))
+    data = np.fromstring(data, np.uint8)
+    data = cv2.imdecode(data, cv2.IMREAD_COLOR)
+    return data