Hough circle transform based blink detection

examples/blinkdet/blinkdet.go

@@ -0,0 +1,138 @@
+package main
+
+import "C"
+
+import (
+	"io/ioutil"
+	"log"
+	"runtime"
+	"unsafe"
+
+	pigo "github.com/esimov/pigo/core"
+)
+
+var (
+	cascade          []byte
+	puplocCascade    []byte
+	faceClassifier   *pigo.Pigo
+	puplocClassifier *pigo.PuplocCascade
+	imageParams      *pigo.ImageParams
+	err              error
+)
+
+func main() {}
+
+//export FindFaces
+func FindFaces(pixels []uint8) uintptr {
+	pointCh := make(chan uintptr)
+
+	results := clusterDetection(pixels, 480, 640)
+	dets := make([][]int, len(results))
+
+	for i := 0; i < len(results); i++ {
+		dets[i] = append(dets[i], results[i].Row, results[i].Col, results[i].Scale, int(results[i].Q), 1)
+		// left eye
+		puploc := &pigo.Puploc{
+			Row:      results[i].Row - int(0.085*float32(results[i].Scale)),
+			Col:      results[i].Col - int(0.185*float32(results[i].Scale)),
+			Scale:    float32(results[i].Scale) * 0.4,
+			Perturbs: 50,
+		}
+		det := puplocClassifier.RunDetector(*puploc, *imageParams)
+		if det.Row > 0 && det.Col > 0 {
+			dets[i] = append(dets[i], det.Row, det.Col, int(det.Scale), int(results[i].Q), 0)
+		}
+
+		// right eye
+		puploc = &pigo.Puploc{
+			Row:      results[i].Row - int(0.085*float32(results[i].Scale)),
+			Col:      results[i].Col + int(0.185*float32(results[i].Scale)),
+			Scale:    float32(results[i].Scale) * 0.4,
+			Perturbs: 50,
+		}
+
+		det = puplocClassifier.RunDetector(*puploc, *imageParams)
+		if det.Row > 0 && det.Col > 0 {
+			dets[i] = append(dets[i], det.Row, det.Col, int(det.Scale), int(results[i].Q), 0)
+		}
+	}
+
+	coords := make([]int, 0, len(dets))
+	go func() {
+		// Since in Go we cannot transfer a 2d array trough an array pointer
+		// we have to transform it into 1d array.
+		for _, v := range dets {
+			coords = append(coords, v...)
+		}
+
+		// Include as a first slice element the number of detected faces.
+		// We need to transfer this value in order to define the Python array buffer length.
+		coords = append([]int{len(dets), 0, 0, 0, 0}, coords...)
+
+		// Convert the slice into an array pointer.
+		s := *(*[]uint8)(unsafe.Pointer(&coords))
+		p := uintptr(unsafe.Pointer(&s[0]))
+
+		// Ensure `det` is not freed up by GC prematurely.
+		runtime.KeepAlive(coords)
+
+		// return the pointer address
+		pointCh <- p
+	}()
+	return <-pointCh
+}
+
+// clusterDetection runs Pigo face detector core methods
+// and returns a cluster with the detected faces coordinates.
+func clusterDetection(pixels []uint8, rows, cols int) []pigo.Detection {
+	imageParams = &pigo.ImageParams{
+		Pixels: pixels,
+		Rows:   rows,
+		Cols:   cols,
+		Dim:    cols,
+	}
+	cParams := pigo.CascadeParams{
+		MinSize:     200,
+		MaxSize:     640,
+		ShiftFactor: 0.1,
+		ScaleFactor: 1.1,
+		ImageParams: *imageParams,
+	}
+
+	// Ensure that the face detection classifier is loaded only once.
+	if len(cascade) == 0 {
+		cascade, err = ioutil.ReadFile("../../cascade/facefinder")
+		if err != nil {
+			log.Fatalf("Error reading the cascade file: %v", err)
+		}
+		p := pigo.NewPigo()
+
+		// Unpack the binary file. This will return the number of cascade trees,
+		// the tree depth, the threshold and the prediction from tree's leaf nodes.
+		faceClassifier, err = p.Unpack(cascade)
+		if err != nil {
+			log.Fatalf("Error unpacking the cascade file: %s", err)
+		}
+	}
+
+	// Ensure that we load the pupil localization cascade only once
+	if len(puplocCascade) == 0 {
+		puplocCascade, err := ioutil.ReadFile("../../cascade/puploc")
+		if err != nil {
+			log.Fatalf("Error reading the puploc cascade file: %s", err)
+		}
+		puplocClassifier, err = puplocClassifier.UnpackCascade(puplocCascade)
+		if err != nil {
+			log.Fatalf("Error unpacking the puploc cascade file: %s", err)
+		}
+	}
+
+	// Run the classifier over the obtained leaf nodes and return the detection results.
+	// The result contains quadruplets representing the row, column, scale and detection score.
+	dets := faceClassifier.RunCascade(cParams, 0.0)
+
+	// Calculate the intersection over union (IoU) of two clusters.
+	dets = faceClassifier.ClusterDetections(dets, 0.0)
+
+	return dets
+}

examples/blinkdet/blinkdet.h

@@ -0,0 +1,76 @@
+/* Code generated by cmd/cgo; DO NOT EDIT. */
+
+/* package command-line-arguments */
+
+
+#line 1 "cgo-builtin-export-prolog"
+
+#include <stddef.h> /* for ptrdiff_t below */
+
+#ifndef GO_CGO_EXPORT_PROLOGUE_H
+#define GO_CGO_EXPORT_PROLOGUE_H
+
+#ifndef GO_CGO_GOSTRING_TYPEDEF
+typedef struct { const char *p; ptrdiff_t n; } _GoString_;
+#endif
+
+#endif
+
+/* Start of preamble from import "C" comments.  */
+
+
+
+
+/* End of preamble from import "C" comments.  */
+
+
+/* Start of boilerplate cgo prologue.  */
+#line 1 "cgo-gcc-export-header-prolog"
+
+#ifndef GO_CGO_PROLOGUE_H
+#define GO_CGO_PROLOGUE_H
+
+typedef signed char GoInt8;
+typedef unsigned char GoUint8;
+typedef short GoInt16;
+typedef unsigned short GoUint16;
+typedef int GoInt32;
+typedef unsigned int GoUint32;
+typedef long long GoInt64;
+typedef unsigned long long GoUint64;
+typedef GoInt64 GoInt;
+typedef GoUint64 GoUint;
+typedef __SIZE_TYPE__ GoUintptr;
+typedef float GoFloat32;
+typedef double GoFloat64;
+typedef float _Complex GoComplex64;
+typedef double _Complex GoComplex128;
+
+/*
+  static assertion to make sure the file is being used on architecture
+  at least with matching size of GoInt.
+*/
+typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1];
+
+#ifndef GO_CGO_GOSTRING_TYPEDEF
+typedef _GoString_ GoString;
+#endif
+typedef void *GoMap;
+typedef void *GoChan;
+typedef struct { void *t; void *v; } GoInterface;
+typedef struct { void *data; GoInt len; GoInt cap; } GoSlice;
+
+#endif
+
+/* End of boilerplate cgo prologue.  */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern GoUintptr FindFaces(GoSlice p0);
+
+#ifdef __cplusplus
+}
+#endif

examples/blinkdet/blinkdet.py

@@ -0,0 +1,125 @@
+from ctypes import *
+
+import subprocess
+import numpy as np
+import os
+import cv2
+import time
+
+os.system('go build -o blinkdet.so -buildmode=c-shared blinkdet.go')
+pigo = cdll.LoadLibrary('./blinkdet.so')
+os.system('rm blinkdet.so')
+
+MAX_NDETS = 2024
+ARRAY_DIM = 6
+
+# define class GoPixelSlice to map to:
+# C type struct { void *data; GoInt len; GoInt cap; }
+class GoPixelSlice(Structure):
+	_fields_ = [
+		("pixels", POINTER(c_ubyte)), ("len", c_longlong), ("cap", c_longlong),
+	]
+
+# Obtain the camera pixels and transfer them to Go trough Ctypes.
+def process_frame(pixs):
+	dets = np.zeros(ARRAY_DIM * MAX_NDETS, dtype=np.float32)
+	pixels = cast((c_ubyte * len(pixs))(*pixs), POINTER(c_ubyte))
+	
+	# call FindFaces
+	faces = GoPixelSlice(pixels, len(pixs), len(pixs))
+	pigo.FindFaces.argtypes = [GoPixelSlice]
+	pigo.FindFaces.restype = c_void_p
+
+	# Call the exported FindFaces function from Go. 
+	ndets = pigo.FindFaces(faces)
+	data_pointer = cast(ndets, POINTER((c_longlong * ARRAY_DIM) * MAX_NDETS))
+	
+	if data_pointer :
+		buffarr = ((c_longlong * ARRAY_DIM) * MAX_NDETS).from_address(addressof(data_pointer.contents))
+		res = np.ndarray(buffer=buffarr, dtype=c_longlong, shape=(MAX_NDETS, 5,))
+
+		# The first value of the buffer aray represents the buffer length.
+		dets_len = res[0][0]
+		res = np.delete(res, 0, 0) # delete the first element from the array
+
+		# We have to consider the pupil pair added into the list.
+		# That's why we are multiplying the detection length with 3.
+		dets = list(res.reshape(-1, 5))[0:dets_len*3]
+		return dets
+
+# initialize the camera
+cap = cv2.VideoCapture(0)
+cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
+cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
+
+# Changing the camera resolution introduce a short delay in the camera initialization. 
+# For this reason we should delay the object detection process with a few milliseconds.
+time.sleep(0.4)
+
+showPupil = True
+showEyes = False
+
+while(True):
+	ret, frame = cap.read()
+	pixs = np.ascontiguousarray(frame[:, :, 1].reshape((frame.shape[0], frame.shape[1])))
+	pixs = pixs.flatten()
+
+	# Verify if camera is intialized by checking if pixel array is not empty.
+	if np.any(pixs):
+		dets = process_frame(pixs) # pixs needs to be numpy.uint8 array
+
+		if dets is not None:
+			# We know that the detected faces are taking place in the first positions of the multidimensional array.
+			for det in dets:
+				if det[4] == 1: # 1 == face; 0 == pupil
+					cv2.rectangle(frame, 
+							(int(det[1])-int(det[2]/2), int(det[0])-int(det[2]/2)), 
+							(int(det[1])+int(det[2]/2), int(det[0])+int(det[2]/2)), 
+							(0, 0, 255), 2
+						)
+				else:
+					if showPupil:
+						x1, x2 = int(det[0])-int(det[2]*1.2), int(det[0])+int(det[2]*1.2)
+						y1, y2 = int(det[1])-int(det[2]*1.2), int(det[1])+int(det[2]*1.2)
+						subimg = frame[x1:x2, y1:y2]
+
+						if subimg is not None:
+							gray = cv2.cvtColor(subimg, cv2.COLOR_BGR2GRAY)
+							img_blur = cv2.medianBlur(gray, 3)
+
+							if img_blur is not None:
+								max_radius = int(det[2]*0.45)
+								circles = cv2.HoughCircles(img_blur, cv2.HOUGH_GRADIENT, 1, int(det[2]*0.3), 
+									param1=60, param2=18, minRadius=4, maxRadius=max_radius)
+								
+								if circles is not None:
+									circles = np.uint16(np.around(circles))
+									for i in circles[0, :]:
+										if i[2] < max_radius and i[2] > 0:
+											# Draw outer circle
+											print(i)
+											cv2.circle(frame, (int(det[1]), int(det[0])), i[2], (0, 255, 0), 2)
+											# Draw inner circle
+											cv2.circle(frame, (int(det[1]), int(det[0])), 2, (255, 0, 255), 3)
+											
+						cv2.circle(frame, (int(det[1]), int(det[0])), 4, (0, 0, 255), -1, 8, 0)
+
+					if showEyes:
+						cv2.rectangle(frame, 
+							(int(det[1])-int(det[2]), int(det[0])-int(det[2])), 
+							(int(det[1])+int(det[2]), int(det[0])+int(det[2])), 
+							(0, 255, 0), 2
+						)
+
+	cv2.imshow('', frame)
+
+	key = cv2.waitKey(1)
+	if key & 0xFF == ord('q'):
+		break
+	elif key & 0xFF == ord('w'):
+		showPupil = not showPupil
+	elif key & 0xFF == ord('e'):
+		showEyes = not showEyes
+
+cap.release()
+cv2.destroyAllWindows()