feat(face): add eye status detecter

README.md

@@ -34,6 +34,8 @@ cmake .. # optional -DNCNN_VULKAN=OFF -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COM
   - tracker (for face IOU calculation bettween frames)
   - hopenet (for head pose detection) [Google Drive](https://drive.google.com/drive/folders/1zLam-8s9ZMPDUxUEtNU2F9yFTDRM5fk-?usp=sharing)
   - hair (for hair segmentation) [Google Drive](https://drive.google.com/drive/folders/14DOBaFrxTL1k4T1ved5qfRUUziurItT8?usp=sharing)
+  - eye
+    - lenet (eye status detector) [Google Drive](https://drive.google.com/drive/folders/1jaonx6PeXFLA8gBKo4eQGuxsncVnqS7o?usp=sharing)
 - pose
   - detector (for pose detection/estimation)
     - ultralight [Google Drive](https://drive.google.com/drive/folders/15b-I5HDyGe2WLb-TO85SJYmnYONvGOKh?usp=sharing)

go/README.md

@@ -46,6 +46,8 @@ make -j 4
   - tracker (for face IOU calculation bettween frames)
   - hopenet (for head pose detection) [Google Drive](https://drive.google.com/drive/folders/1zLam-8s9ZMPDUxUEtNU2F9yFTDRM5fk-?usp=sharing)
   - hair (for hair segmentation) [Google Drive](https://drive.google.com/drive/folders/14DOBaFrxTL1k4T1ved5qfRUUziurItT8?usp=sharing)
+  - eye
+    - lenet (eye status detector) [Google Drive](https://drive.google.com/drive/folders/1jaonx6PeXFLA8gBKo4eQGuxsncVnqS7o?usp=sharing)
 - pose
   - detector (for pose detection/estimation)
     - ultralight [Google Drive](https://drive.google.com/drive/folders/15b-I5HDyGe2WLb-TO85SJYmnYONvGOKh?usp=sharing)

go/face/eye/cgo.go

@@ -0,0 +1,11 @@
+// +build !vulkan
+
+package recognizer
+
+/*
+#cgo CXXFLAGS:   --std=c++11 -fopenmp
+#cgo CPPFLAGS:   -I ${SRCDIR}/../../../include -I /usr/local/include
+#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision
+#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../../lib
+*/
+import "C"

go/face/eye/cgo_vulkan.go

@@ -0,0 +1,11 @@
+// +build vulkan
+
+package eye
+
+/*
+#cgo CXXFLAGS:   --std=c++11 -fopenmp
+#cgo CPPFLAGS:   -I ${SRCDIR}/../../../include -I /usr/local/include
+#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision -lglslang -lvulkan -lSPIRV -lOGLCompiler -lMachineIndependent -lGenericCodeGen -lOSDependent
+#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../../lib
+*/
+import "C"

go/face/eye/detecter.go

@@ -0,0 +1,43 @@
+package eye
+
+/*
+#include <stdlib.h>
+#include <stdbool.h>
+#include "openvision/common/common.h"
+#include "openvision/face/eye.h"
+*/
+import "C"
+import (
+	"unsafe"
+
+	openvision "github.com/bububa/openvision/go"
+	"github.com/bububa/openvision/go/common"
+)
+
+// Detecter represents Eye Detector interface
+type Detecter interface {
+	common.Estimator
+	Status(img *common.Image, face common.Rectangle) ([]float64, error)
+}
+
+// Status extract scores using recognizer
+func Status(r Detecter, img *common.Image, faceRect common.Rectangle) ([]float64, error) {
+	imgWidth := img.WidthF64()
+	imgHeight := img.HeightF64()
+	data := img.Bytes()
+	CFeatures := common.NewCFloatVector()
+	defer common.FreeCFloatVector(CFeatures)
+	CRect := faceRect.CRect(imgWidth, imgHeight)
+	errCode := C.eye_status(
+		(C.IEye)(r.Pointer()),
+		(*C.uchar)(unsafe.Pointer(&data[0])),
+		C.int(imgWidth), C.int(imgHeight),
+		(*C.Rect)(unsafe.Pointer(CRect)),
+		(*C.FloatVector)(unsafe.Pointer(CFeatures)),
+	)
+	C.free(unsafe.Pointer(CRect))
+	if errCode != 0 {
+		return nil, openvision.RecognizeFaceError(int(errCode))
+	}
+	return common.GoFloatVector(CFeatures), nil
+}

go/face/eye/doc.go

@@ -0,0 +1,2 @@
+// Package eye include eye status detector 
+package eye 

go/face/eye/lenet.go

@@ -0,0 +1,45 @@
+package eye
+
+/*
+#include <stdlib.h>
+#include <stdbool.h>
+#include "openvision/face/eye.h"
+*/
+import "C"
+import (
+	"unsafe"
+
+	"github.com/bububa/openvision/go/common"
+)
+
+// Lenet represents Lenet detecter
+type Lenet struct {
+	d C.IEye
+}
+
+// NetLenet returns a new Lenet detecter
+func NewLenet() *Lenet {
+	return &Lenet{
+		d: C.new_lenet_eye(),
+	}
+}
+
+// Pointer implement Estimator interface
+func (d *Lenet) Pointer() unsafe.Pointer {
+	return unsafe.Pointer(d.d)
+}
+
+// LoadModel implement Recognizer interface
+func (d *Lenet) LoadModel(modelPath string) error {
+	return common.EstimatorLoadModel(d, modelPath)
+}
+
+// Destroy implement Recognizer interface
+func (d *Lenet) Destroy() {
+	common.DestroyEstimator(d)
+}
+
+// Status implement Eye Detecter interface
+func (d *Lenet) Status(img *common.Image, faceRect common.Rectangle) ([]float64, error) {
+	return Status(d, img, faceRect)
+}

go/face/tracker/cgo.go

@@ -1,6 +1,6 @@
 // +build !vulkan
 
-package tracker
+package eye
 
 /*
 #cgo CXXFLAGS:   --std=c++11 -fopenmp

src/CMakeLists.txt

@@ -58,6 +58,7 @@ target_include_directories(openvision
 
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/face/tracker>
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/face/hair>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/face/eye>
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/face/hopenet>
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/face/aligner>
 
@@ -92,6 +93,7 @@ file(COPY
     ${CMAKE_CURRENT_SOURCE_DIR}/face/hopenet.h
     ${CMAKE_CURRENT_SOURCE_DIR}/face/aligner.h
     ${CMAKE_CURRENT_SOURCE_DIR}/face/hair.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/face/eye.h
     DESTINATION ${INCLUDE_OUTPUT_PATH}/openvision/face
 )
 

src/face/eye.h

@@ -0,0 +1,17 @@
+#ifndef _FACE_EYE_C_H_
+#define _FACE_EYE_C_H_
+
+#include "common.h"
+
+#ifdef __cplusplus
+#include "eye/eye.hpp"
+extern "C" {
+#endif
+typedef void *IEye;
+IEye new_lenet_eye();
+int eye_status(IEye d, const unsigned char *rgbdata, int img_width,
+               int img_height, const Rect *rect, FloatVector *cls_scores);
+#ifdef __cplusplus
+}
+#endif
+#endif // !_FACE_HAIR_C_H_

src/face/eye/eye.cpp

@@ -0,0 +1,20 @@
+#include "../eye.h"
+#include "lenet/lenet.hpp"
+
+IEye new_lenet_eye() { return new ovface::LenetEye(); }
+
+int eye_status(IEye d, const unsigned char *rgbdata, int img_width,
+               int img_height, const Rect *rect, FloatVector *cls_scores) {
+  std::vector<float> scores;
+  int ret = static_cast<ovface::Eye *>(d)->Status(rgbdata, img_width,
+                                                  img_height, *rect, scores);
+  if (ret != 0) {
+    return ret;
+  }
+  cls_scores->length = scores.size();
+  cls_scores->values = (float *)malloc(cls_scores->length * sizeof(float));
+  for (int i = 0; i < cls_scores->length; ++i) {
+    cls_scores->values[i] = scores[i];
+  }
+  return 0;
+}

src/face/eye/eye.hpp

@@ -0,0 +1,13 @@
+#ifndef _FACE_EYE_H_
+#define _FACE_EYE_H_
+
+#include "../../common/common.hpp"
+namespace ovface {
+class Eye : public ov::Estimator {
+public:
+  virtual int Status(const unsigned char *rgbdata, int img_width,
+                     int img_height, const ov::Rect rect,
+                     std::vector<float> &cls_scores) = 0;
+};
+} // namespace ovface
+#endif // !_FACE_EYE_H_

src/face/eye/lenet/lenet.cpp

@@ -0,0 +1,39 @@
+#include "lenet.hpp"
+
+#ifdef OV_VULKAN
+#include "gpu.h"
+#endif // OV_VULKAN
+
+namespace ovface {
+int LenetEye::Status(const unsigned char *rgbdata, int img_width,
+                     int img_height, const ov::Rect rect,
+                     std::vector<float> &cls_scores) {
+  if (!initialized_) {
+    return 10000;
+  }
+  if (rgbdata == 0) {
+    return 10001;
+  }
+  cls_scores.clear();
+  ncnn::Mat in = ncnn::Mat::from_pixels_roi_resize(
+      rgbdata, ncnn::Mat::PIXEL_RGB2GRAY, img_width, img_height, rect.x, rect.y,
+      rect.width, rect.height, target_width, target_height); // PIXEL_GRAY
+  in.substract_mean_normalize(mean_vals, 0);
+  ncnn::Extractor ex = net_->create_extractor();
+  ex.set_light_mode(light_mode_);
+  ex.set_num_threads(num_threads);
+  ex.input("data", in);
+
+  ncnn::Mat out;
+  ex.extract("prob", out);
+
+  cls_scores.resize(out.c);
+  for (int c = 0; c < out.c; c++) {
+    const float *prob = out.channel(c);
+    cls_scores[c] = prob[0];
+  }
+
+  return 0;
+}
+
+} // namespace ovface

src/face/eye/lenet/lenet.hpp

@@ -0,0 +1,17 @@
+#ifndef _FACE_EYE_LENET_H_
+#define _FACE_EYE_LENET_H_
+#include "../eye.hpp"
+
+namespace ovface {
+class LenetEye : public Eye {
+public:
+  int Status(const unsigned char *rgbdata, int img_width, int img_height,
+             const ov::Rect rect, std::vector<float> &cls_scores);
+
+private:
+  const int target_width = 36;
+  const int target_height = 28;
+  const float mean_vals[1] = {60.f};
+};
+} // namespace ovface
+#endif // !_FACE_EYE_LENET_H_