diff --git a/README.md b/README.md index 59c75e9..7bdd285 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ cmake .. # optional -DNCNN_VULKAN=OFF -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COM - tracker (for face IOU calculation bettween frames) - hopenet (for head pose detection) [Google Drive](https://drive.google.com/drive/folders/1zLam-8s9ZMPDUxUEtNU2F9yFTDRM5fk-?usp=sharing) - hair (for hair segmentation) [Google Drive](https://drive.google.com/drive/folders/14DOBaFrxTL1k4T1ved5qfRUUziurItT8?usp=sharing) + - eye + - lenet (eye status detector) [Google Drive](https://drive.google.com/drive/folders/1jaonx6PeXFLA8gBKo4eQGuxsncVnqS7o?usp=sharing) - pose - detector (for pose detection/estimation) - ultralight [Google Drive](https://drive.google.com/drive/folders/15b-I5HDyGe2WLb-TO85SJYmnYONvGOKh?usp=sharing) diff --git a/go/README.md b/go/README.md index ba72e51..b60f548 100644 --- a/go/README.md +++ b/go/README.md @@ -46,6 +46,8 @@ make -j 4 - tracker (for face IOU calculation bettween frames) - hopenet (for head pose detection) [Google Drive](https://drive.google.com/drive/folders/1zLam-8s9ZMPDUxUEtNU2F9yFTDRM5fk-?usp=sharing) - hair (for hair segmentation) [Google Drive](https://drive.google.com/drive/folders/14DOBaFrxTL1k4T1ved5qfRUUziurItT8?usp=sharing) + - eye + - lenet (eye status detector) [Google Drive](https://drive.google.com/drive/folders/1jaonx6PeXFLA8gBKo4eQGuxsncVnqS7o?usp=sharing) - pose - detector (for pose detection/estimation) - ultralight [Google Drive](https://drive.google.com/drive/folders/15b-I5HDyGe2WLb-TO85SJYmnYONvGOKh?usp=sharing) diff --git a/go/face/eye/cgo.go b/go/face/eye/cgo.go new file mode 100644 index 0000000..05f982e --- /dev/null +++ b/go/face/eye/cgo.go @@ -0,0 +1,11 @@ +// +build !vulkan + +package recognizer + +/* +#cgo CXXFLAGS: --std=c++11 -fopenmp +#cgo CPPFLAGS: -I ${SRCDIR}/../../../include -I /usr/local/include +#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision +#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../../lib +*/ +import "C" diff --git a/go/face/eye/cgo_vulkan.go b/go/face/eye/cgo_vulkan.go new file mode 100644 index 0000000..a72bb5c --- /dev/null +++ b/go/face/eye/cgo_vulkan.go @@ -0,0 +1,11 @@ +// +build vulkan + +package eye + +/* +#cgo CXXFLAGS: --std=c++11 -fopenmp +#cgo CPPFLAGS: -I ${SRCDIR}/../../../include -I /usr/local/include +#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision -lglslang -lvulkan -lSPIRV -lOGLCompiler -lMachineIndependent -lGenericCodeGen -lOSDependent +#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../../lib +*/ +import "C" diff --git a/go/face/eye/detecter.go b/go/face/eye/detecter.go new file mode 100644 index 0000000..cd973a2 --- /dev/null +++ b/go/face/eye/detecter.go @@ -0,0 +1,43 @@ +package eye + +/* +#include +#include +#include "openvision/common/common.h" +#include "openvision/face/eye.h" +*/ +import "C" +import ( + "unsafe" + + openvision "github.com/bububa/openvision/go" + "github.com/bububa/openvision/go/common" +) + +// Detecter represents Eye Detector interface +type Detecter interface { + common.Estimator + Status(img *common.Image, face common.Rectangle) ([]float64, error) +} + +// Status extract scores using recognizer +func Status(r Detecter, img *common.Image, faceRect common.Rectangle) ([]float64, error) { + imgWidth := img.WidthF64() + imgHeight := img.HeightF64() + data := img.Bytes() + CFeatures := common.NewCFloatVector() + defer common.FreeCFloatVector(CFeatures) + CRect := faceRect.CRect(imgWidth, imgHeight) + errCode := C.eye_status( + (C.IEye)(r.Pointer()), + (*C.uchar)(unsafe.Pointer(&data[0])), + C.int(imgWidth), C.int(imgHeight), + (*C.Rect)(unsafe.Pointer(CRect)), + (*C.FloatVector)(unsafe.Pointer(CFeatures)), + ) + C.free(unsafe.Pointer(CRect)) + if errCode != 0 { + return nil, openvision.RecognizeFaceError(int(errCode)) + } + return common.GoFloatVector(CFeatures), nil +} diff --git a/go/face/eye/doc.go b/go/face/eye/doc.go new file mode 100644 index 0000000..15a80c4 --- /dev/null +++ b/go/face/eye/doc.go @@ -0,0 +1,2 @@ +// Package eye include eye status detector +package eye diff --git a/go/face/eye/lenet.go b/go/face/eye/lenet.go new file mode 100644 index 0000000..a1b4f10 --- /dev/null +++ b/go/face/eye/lenet.go @@ -0,0 +1,45 @@ +package eye + +/* +#include +#include +#include "openvision/face/eye.h" +*/ +import "C" +import ( + "unsafe" + + "github.com/bububa/openvision/go/common" +) + +// Lenet represents Lenet detecter +type Lenet struct { + d C.IEye +} + +// NetLenet returns a new Lenet detecter +func NewLenet() *Lenet { + return &Lenet{ + d: C.new_lenet_eye(), + } +} + +// Pointer implement Estimator interface +func (d *Lenet) Pointer() unsafe.Pointer { + return unsafe.Pointer(d.d) +} + +// LoadModel implement Recognizer interface +func (d *Lenet) LoadModel(modelPath string) error { + return common.EstimatorLoadModel(d, modelPath) +} + +// Destroy implement Recognizer interface +func (d *Lenet) Destroy() { + common.DestroyEstimator(d) +} + +// Status implement Eye Detecter interface +func (d *Lenet) Status(img *common.Image, faceRect common.Rectangle) ([]float64, error) { + return Status(d, img, faceRect) +} diff --git a/go/face/tracker/cgo.go b/go/face/tracker/cgo.go index 0f33239..c64d9fc 100644 --- a/go/face/tracker/cgo.go +++ b/go/face/tracker/cgo.go @@ -1,6 +1,6 @@ // +build !vulkan -package tracker +package eye /* #cgo CXXFLAGS: --std=c++11 -fopenmp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6b87012..99be3f4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,6 +58,7 @@ target_include_directories(openvision $ $ + $ $ $ @@ -92,6 +93,7 @@ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/face/hopenet.h ${CMAKE_CURRENT_SOURCE_DIR}/face/aligner.h ${CMAKE_CURRENT_SOURCE_DIR}/face/hair.h + ${CMAKE_CURRENT_SOURCE_DIR}/face/eye.h DESTINATION ${INCLUDE_OUTPUT_PATH}/openvision/face ) diff --git a/src/face/eye.h b/src/face/eye.h new file mode 100644 index 0000000..89c5072 --- /dev/null +++ b/src/face/eye.h @@ -0,0 +1,17 @@ +#ifndef _FACE_EYE_C_H_ +#define _FACE_EYE_C_H_ + +#include "common.h" + +#ifdef __cplusplus +#include "eye/eye.hpp" +extern "C" { +#endif +typedef void *IEye; +IEye new_lenet_eye(); +int eye_status(IEye d, const unsigned char *rgbdata, int img_width, + int img_height, const Rect *rect, FloatVector *cls_scores); +#ifdef __cplusplus +} +#endif +#endif // !_FACE_HAIR_C_H_ diff --git a/src/face/eye/eye.cpp b/src/face/eye/eye.cpp new file mode 100644 index 0000000..e76b141 --- /dev/null +++ b/src/face/eye/eye.cpp @@ -0,0 +1,20 @@ +#include "../eye.h" +#include "lenet/lenet.hpp" + +IEye new_lenet_eye() { return new ovface::LenetEye(); } + +int eye_status(IEye d, const unsigned char *rgbdata, int img_width, + int img_height, const Rect *rect, FloatVector *cls_scores) { + std::vector scores; + int ret = static_cast(d)->Status(rgbdata, img_width, + img_height, *rect, scores); + if (ret != 0) { + return ret; + } + cls_scores->length = scores.size(); + cls_scores->values = (float *)malloc(cls_scores->length * sizeof(float)); + for (int i = 0; i < cls_scores->length; ++i) { + cls_scores->values[i] = scores[i]; + } + return 0; +} diff --git a/src/face/eye/eye.hpp b/src/face/eye/eye.hpp new file mode 100644 index 0000000..2da098f --- /dev/null +++ b/src/face/eye/eye.hpp @@ -0,0 +1,13 @@ +#ifndef _FACE_EYE_H_ +#define _FACE_EYE_H_ + +#include "../../common/common.hpp" +namespace ovface { +class Eye : public ov::Estimator { +public: + virtual int Status(const unsigned char *rgbdata, int img_width, + int img_height, const ov::Rect rect, + std::vector &cls_scores) = 0; +}; +} // namespace ovface +#endif // !_FACE_EYE_H_ diff --git a/src/face/eye/lenet/lenet.cpp b/src/face/eye/lenet/lenet.cpp new file mode 100644 index 0000000..1332535 --- /dev/null +++ b/src/face/eye/lenet/lenet.cpp @@ -0,0 +1,39 @@ +#include "lenet.hpp" + +#ifdef OV_VULKAN +#include "gpu.h" +#endif // OV_VULKAN + +namespace ovface { +int LenetEye::Status(const unsigned char *rgbdata, int img_width, + int img_height, const ov::Rect rect, + std::vector &cls_scores) { + if (!initialized_) { + return 10000; + } + if (rgbdata == 0) { + return 10001; + } + cls_scores.clear(); + ncnn::Mat in = ncnn::Mat::from_pixels_roi_resize( + rgbdata, ncnn::Mat::PIXEL_RGB2GRAY, img_width, img_height, rect.x, rect.y, + rect.width, rect.height, target_width, target_height); // PIXEL_GRAY + in.substract_mean_normalize(mean_vals, 0); + ncnn::Extractor ex = net_->create_extractor(); + ex.set_light_mode(light_mode_); + ex.set_num_threads(num_threads); + ex.input("data", in); + + ncnn::Mat out; + ex.extract("prob", out); + + cls_scores.resize(out.c); + for (int c = 0; c < out.c; c++) { + const float *prob = out.channel(c); + cls_scores[c] = prob[0]; + } + + return 0; +} + +} // namespace ovface diff --git a/src/face/eye/lenet/lenet.hpp b/src/face/eye/lenet/lenet.hpp new file mode 100644 index 0000000..8bdafaf --- /dev/null +++ b/src/face/eye/lenet/lenet.hpp @@ -0,0 +1,17 @@ +#ifndef _FACE_EYE_LENET_H_ +#define _FACE_EYE_LENET_H_ +#include "../eye.hpp" + +namespace ovface { +class LenetEye : public Eye { +public: + int Status(const unsigned char *rgbdata, int img_width, int img_height, + const ov::Rect rect, std::vector &cls_scores); + +private: + const int target_width = 36; + const int target_height = 28; + const float mean_vals[1] = {60.f}; +}; +} // namespace ovface +#endif // !_FACE_EYE_LENET_H_