commit 4f7a0f669214d1c2df3964da27872485b736c126 Author: Syd Xu Date: Thu Oct 21 18:30:38 2021 +0800 init commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..049c503 --- /dev/null +++ b/.gitignore @@ -0,0 +1,72 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# Folders +_obj +_test + +# CMakefiles +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps +.cache/ +.cmake/ + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +.DS_Store +test +.vim +dist/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..8f3d4f9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ncnn"] + path = ncnn + url = https://github.com/Tencent/ncnn.git diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..28adff8 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.1) +project(openvision) + +set(CMAKE_BUILD_TYPE Release) +set(PROJECT_SOURCE_DIR, src) +set(PROJECT_BINARY_DIR, build) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) +set(CMAKE_BUILD_FILES_DIRECTORY ${PROJECT_BINARY_DIR}) +set(CMAKE_BUILD_DIRECTORY ${PROJECT_BINARY_DIR}) +set(CMAKE_BINARY_DIR ${PROJECT_BINARY_DIR}) +set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/../bin) +set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/../lib) +set(INCLUDE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/../include) +set(CMAKE_CACHEFILE_DIR ${PROJECT_BINARY_DIR}) + +Option(MIRROR_OPENMP "openmp support" ON) +Option(MIRROR_VULKAN "vulkan compute used" ON) +Option(MIRROR_BUILD_CLASSIFIER "build classifier test" ON) + +add_subdirectory(ncnn) + +add_subdirectory(src) +# add_subdirectory(examples) diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000..94548af --- /dev/null +++ b/build/.gitignore @@ -0,0 +1,3 @@ +* +*/ +!.gitignore diff --git a/data/images/.gitignore b/data/images/.gitignore new file mode 100644 index 0000000..94548af --- /dev/null +++ b/data/images/.gitignore @@ -0,0 +1,3 @@ +* +*/ +!.gitignore diff --git a/data/models/.gitignore b/data/models/.gitignore new file mode 100644 index 0000000..94548af --- /dev/null +++ b/data/models/.gitignore @@ -0,0 +1,3 @@ +* +*/ +!.gitignore diff --git a/go/common/cgo.go b/go/common/cgo.go new file mode 100644 index 0000000..9086aad --- /dev/null +++ b/go/common/cgo.go @@ -0,0 +1,36 @@ +package common + +/* +#cgo CXXFLAGS: --std=c++11 -fopenmp +#cgo CPPFLAGS: -I ${SRCDIR}/../../include -I /usr/local/include +#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision +#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../lib +#include +#include +#include "openvision/common/common.h" +*/ +import "C" +import "unsafe" + +// NewCFloatVector returns C.FloatVector pointer +func NewCFloatVector() *C.FloatVector { + return (*C.FloatVector)(C.malloc(C.sizeof_FloatVector)) +} + +// GoFloatVector convert C.FloatVector to []float64 +func GoFloatVector(cVector *C.FloatVector) []float64 { + l := int(cVector.length) + ret := make([]float64, 0, l) + ptr := unsafe.Pointer(cVector.values) + for i := 0; i < l; i++ { + v := (*float32)(unsafe.Pointer(uintptr(ptr) + uintptr(C.int(i)*C.sizeof_float))) + ret = append(ret, float64(*v)) + } + return ret +} + +// FreeCFloatVector release C.FloatVector memory +func FreeCFloatVector(c *C.FloatVector) { + C.FreeFloatVector(c) + C.free(unsafe.Pointer(c)) +} diff --git a/go/common/color.go b/go/common/color.go new file mode 100644 index 0000000..1042b4a --- /dev/null +++ b/go/common/color.go @@ -0,0 +1,44 @@ +package common + +import ( + "fmt" + "image/color" + "strings" +) + +// ColorFromHex get color from hex +func ColorFromHex(hexColor string) color.RGBA { + r, g, b, a := parseHexColor(hexColor) + return color.RGBA{uint8(r), uint8(g), uint8(b), uint8(a)} +} + +// parseHexColor parse color hex string to rgba value +func parseHexColor(x string) (r, g, b, a uint32) { + if !strings.HasPrefix(x, "#") { + return color.Transparent.RGBA() + } + x = strings.TrimPrefix(x, "#") + a = 255 + if len(x) == 3 { + format := "%1x%1x%1x" + fmt.Sscanf(x, format, &r, &g, &b) + r |= r << 4 + g |= g << 4 + b |= b << 4 + } + if len(x) == 6 { + format := "%02x%02x%02x" + fmt.Sscanf(x, format, &r, &g, &b) + } + if len(x) == 8 { + format := "%02x%02x%02x%02x" + fmt.Sscanf(x, format, &r, &g, &b, &a) + } + return +} + +const ( + Green = "#64DD17" + Pink = "#E91E63" + Red = "#FF1744" +) diff --git a/go/common/doc.go b/go/common/doc.go new file mode 100644 index 0000000..b566983 --- /dev/null +++ b/go/common/doc.go @@ -0,0 +1,2 @@ +// Package common . +package common diff --git a/go/common/geometry.go b/go/common/geometry.go new file mode 100644 index 0000000..7fa1dde --- /dev/null +++ b/go/common/geometry.go @@ -0,0 +1,91 @@ +package common + +/* +#include +#include +#include "openvision/common/common.h" +*/ +import "C" +import ( + "unsafe" +) + +// Rectangle represents a Rectangle +type Rectangle struct { + X float64 + Y float64 + Width float64 + Height float64 +} + +// Rect returns a Retancle +func Rect(x, y, w, h float64) Rectangle { + return Rectangle{ + X: x, + Y: y, + Width: w, + Height: h, + } +} + +// MaxX returns right x +func (r Rectangle) MaxX() float64 { + return r.X + r.Width +} + +// MaxY returns bottom y +func (r Rectangle) MaxY() float64 { + return r.Y + r.Height +} + +// CRect returns C.Rect +func (r Rectangle) CRect() *C.Rect { + v := (*C.Rect)(C.malloc(C.sizeof_Rect)) + v.x = C.int(r.X) + v.y = C.int(r.Y) + v.width = C.int(r.Width) + v.height = C.int(r.Height) + return v +} + +// Point represents a Point +type Point struct { + X float64 + Y float64 +} + +// Pt returns a New Point +func Pt(x, y float64) Point { + return Point{x, y} +} + +// GoPoint2f conver C.Point2f to Point +func GoPoint2f(c *C.Point2f, w float64, h float64) Point { + return Pt( + float64(c.x)/w, + float64(c.y)/h, + ) +} + +// NewCPoint2fVector retruns C.Point2fVector pointer +func NewCPoint2fVector() *C.Point2fVector { + return (*C.Point2fVector)(C.malloc(C.sizeof_Point2f)) +} + +// GoPoint2fVector convert C.Point2fVector to []Point +func GoPoint2fVector(cVector *C.Point2fVector, w float64, h float64) []Point { + l := int(cVector.length) + ret := make([]Point, 0, l) + ptr := unsafe.Pointer(cVector.points) + for i := 0; i < l; i++ { + cPoint2f := (*C.Point2f)(unsafe.Pointer(uintptr(ptr) + uintptr(C.sizeof_Point2f*C.int(i)))) + ret = append(ret, GoPoint2f(cPoint2f, w, h)) + } + return ret +} + +// FreeCPoint2fVector release C.Point2fVector memory +func FreeCPoint2fVector(c *C.Point2fVector) { + C.FreePoint2fVector(c) + C.free(unsafe.Pointer(c)) +} diff --git a/go/common/image.go b/go/common/image.go new file mode 100644 index 0000000..312a365 --- /dev/null +++ b/go/common/image.go @@ -0,0 +1,153 @@ +package common + +import ( + "bytes" + "image" + "io" + "sync" + + "github.com/llgcode/draw2d/draw2dimg" + "github.com/llgcode/draw2d/draw2dkit" +) + +// Image image with buffer cache +type Image struct { + image.Image + buffer *bytes.Buffer +} + +// NewImage returns a new Image +func NewImage(img image.Image) *Image { + buf := new(bytes.Buffer) + Image2RGB(buf, img) + return &Image{ + Image: img, + buffer: buf, + } +} + +// Bytes returns image bytes in rgb +func (i Image) Bytes() []byte { + if i.buffer == nil { + return nil + } + return i.buffer.Bytes() +} + +// Width returns image width +func (i Image) Width() int { + return i.Bounds().Dx() +} + +// Height returns image height +func (i Image) Height() int { + return i.Bounds().Dy() +} + +// WidthF64 returns image width in float64 +func (i Image) WidthF64() float64 { + return float64(i.Bounds().Dx()) +} + +// HeightF64 returns image height in float64 +func (i Image) HeightF64() float64 { + return float64(i.Bounds().Dy()) +} + +// Crop returns cropped image bytes in Rectangle +func (i Image) Crop(rect Rectangle) []byte { + imgW := i.WidthF64() + imgH := i.HeightF64() + imgWInt := i.Width() + imgHInt := i.Height() + cropWidth := int(rect.Width) + if rect.MaxX() > imgW { + cropWidth = imgWInt + } + cropHeight := int(rect.Height) + if rect.MaxY() > imgH { + cropHeight = imgHInt + } + xOffset := int(rect.X) + if rect.X < 1e-15 { + xOffset = 0 + } + yOffset := int(rect.Y) + if rect.Y < 1e-15 { + yOffset = 0 + } + imgData := i.Bytes() + ret := make([]byte, 0, cropWidth*cropHeight*3) + pool := &sync.Pool{ + New: func() interface{} { + return make([]byte, cropWidth*3) + }, + } + for y := 0; y < cropHeight; y++ { + srcCur := ((y+yOffset)*imgWInt + xOffset) * 3 + dist := pool.Get().([]byte) + dist = dist[:0] + copy(dist, imgData[srcCur:cropWidth*3]) + pool.Put(dist) + ret = append(ret, dist...) + } + return ret +} + +// Image2RGB write image rgbdata to buffer +func Image2RGB(buf io.Writer, img image.Image) { + bounds := img.Bounds() + for j := bounds.Min.Y; j < bounds.Max.Y; j++ { + for i := bounds.Min.X; i < bounds.Max.X; i++ { + r, g, b, _ := img.At(i, j).RGBA() + buf.Write([]byte{byte(b >> 8), byte(g >> 8), byte(r >> 8)}) + } + } +} + +// Image2BGR write image bgrdata to buffer +func Image2BGR(buf io.Writer, img image.Image) { + bounds := img.Bounds() + for j := bounds.Min.Y; j < bounds.Max.Y; j++ { + for i := bounds.Min.X; i < bounds.Max.X; i++ { + r, g, b, _ := img.At(i, j).RGBA() + buf.Write([]byte{byte(r >> 8), byte(g >> 8), byte(b >> 8)}) + } + } +} + +// DrawRectangle draw rectangle on image +func DrawRectangle(gc *draw2dimg.GraphicContext, rect Rectangle, borderColor string, bgColor string, strokeWidth float64) { + gc.SetStrokeColor(ColorFromHex(borderColor)) + if bgColor != "" { + gc.SetFillColor(ColorFromHex(bgColor)) + } + gc.SetLineWidth(strokeWidth) + draw2dkit.Rectangle(gc, rect.X, rect.Y, rect.MaxX(), rect.MaxY()) + if strokeWidth == 0 || bgColor != "" { + if bgColor == "" { + gc.SetFillColor(ColorFromHex(borderColor)) + } + gc.FillStroke() + } else { + gc.Stroke() + } +} + +// DrawCircle draw circle on image +func DrawCircle(gc *draw2dimg.GraphicContext, pt Point, r float64, borderColor string, bgColor string, strokeWidth float64) { + gc.SetStrokeColor(ColorFromHex(borderColor)) + if bgColor != "" { + gc.SetFillColor(ColorFromHex(bgColor)) + } + gc.SetLineWidth(strokeWidth) + draw2dkit.Circle(gc, pt.X, pt.Y, r) + if strokeWidth == 0 || bgColor != "" { + if bgColor == "" { + gc.SetFillColor(ColorFromHex(borderColor)) + } + gc.FillStroke() + } else { + gc.Stroke() + } +} diff --git a/go/doc.go b/go/doc.go new file mode 100644 index 0000000..d29c240 --- /dev/null +++ b/go/doc.go @@ -0,0 +1,2 @@ +// Package openvision libopenvision golang binding +package openvision diff --git a/go/error.go b/go/error.go new file mode 100644 index 0000000..bc14c47 --- /dev/null +++ b/go/error.go @@ -0,0 +1,47 @@ +package openvision + +// Error customed error +type Error struct { + // Code . + Code int + // Message . + Message string +} + +// Error represents error interface +func (e Error) Error() string { + return e.Message +} + +var ( + LoadModelError = func(code int) Error { + return Error{ + Code: code, + Message: "load model failed", + } + } + DetectFaceError = func(code int) Error { + return Error{ + Code: code, + Message: "detect face failed", + } + } + FaceLandmarkError = func(code int) Error { + return Error{ + Code: code, + Message: "face landmark failed", + } + } + RecognizeFaceError = func(code int) Error { + return Error{ + Code: code, + Message: "recognize face failed", + } + } + TrackFaceError = func(code int) Error { + return Error{ + Code: code, + Message: "track face failed", + } + } +) diff --git a/go/examples/detecter/main.go b/go/examples/detecter/main.go new file mode 100644 index 0000000..601460f --- /dev/null +++ b/go/examples/detecter/main.go @@ -0,0 +1,149 @@ +package main + +import ( + "bytes" + "fmt" + "image" + "image/jpeg" + "log" + "os" + "os/user" + "path/filepath" + "strings" + + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face/detecter" + facedrawer "github.com/bububa/openvision/go/face/drawer" +) + +func main() { + wd, _ := os.Getwd() + dataPath := cleanPath(wd, "~/go/src/github.com/bububa/openvision/data") + imgPath := filepath.Join(dataPath, "./images") + modelPath := filepath.Join(dataPath, "./models") + test_detect(imgPath, modelPath) + test_mask(imgPath, modelPath) +} + +func test_detect(imgPath string, modelPath string) { + for idx, d := range []detecter.Detecter{ + retinaface(modelPath), + centerface(modelPath), + mtcnn(modelPath), + } { + detect(d, imgPath, idx, "4.jpg", false) + d.Destroy() + } +} + +func test_mask(imgPath string, modelPath string) { + d := anticonv(modelPath) + defer d.Destroy() + detect(d, imgPath, 0, "mask3.jpg", true) +} + +func retinaface(modelPath string) detecter.Detecter { + modelPath = filepath.Join(modelPath, "fd") + d := detecter.NewRetinaFace() + if err := d.LoadModel(modelPath); err != nil { + log.Fatalln(err) + } + return d +} + +func mtcnn(modelPath string) detecter.Detecter { + modelPath = filepath.Join(modelPath, "mtcnn") + d := detecter.NewMtcnn() + if err := d.LoadModel(modelPath); err != nil { + log.Fatalln(err) + } + return d +} + +func centerface(modelPath string) detecter.Detecter { + modelPath = filepath.Join(modelPath, "centerface") + d := detecter.NewCenterface() + if err := d.LoadModel(modelPath); err != nil { + log.Fatalln(err) + } + return d +} + +func anticonv(modelPath string) detecter.Detecter { + modelPath = filepath.Join(modelPath, "mask") + d := detecter.NewAnticonv() + if err := d.LoadModel(modelPath); err != nil { + log.Fatalln(err) + } + return d +} + +func detect(d detecter.Detecter, imgPath string, idx int, filename string, mask bool) { + inPath := filepath.Join(imgPath, filename) + img, err := loadImage(inPath) + if err != nil { + log.Fatalln("load image failed,", err) + } + faces, err := d.DetectFace(common.NewImage(img)) + if err != nil { + log.Fatalln(err) + } + + outPath := filepath.Join(imgPath, "./results", fmt.Sprintf("%d-%s", idx, filename)) + + var drawer *facedrawer.Drawer + if mask { + drawer = facedrawer.New( + facedrawer.WithBorderColor(common.Red), + facedrawer.WithMaskColor(common.Green), + ) + } else { + drawer = facedrawer.New() + } + + out := drawer.Draw(img, faces) + + if err := saveImage(out, outPath); err != nil { + log.Fatalln(err) + } + + log.Printf("faces: %+v\n", faces) +} + +func loadImage(filePath string) (image.Image, error) { + fn, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer fn.Close() + img, _, err := image.Decode(fn) + if err != nil { + return nil, err + } + return img, nil +} + +func saveImage(img image.Image, filePath string) error { + buf := new(bytes.Buffer) + if err := jpeg.Encode(buf, img, nil); err != nil { + return err + } + fn, err := os.Create(filePath) + if err != nil { + return err + } + defer fn.Close() + fn.Write(buf.Bytes()) + return nil +} + +func cleanPath(wd string, path string) string { + usr, _ := user.Current() + dir := usr.HomeDir + if path == "~" { + return dir + } else if strings.HasPrefix(path, "~/") { + return filepath.Join(dir, path[2:]) + } + return filepath.Join(wd, path) +} diff --git a/go/examples/landmarker/main.go b/go/examples/landmarker/main.go new file mode 100644 index 0000000..6d4b307 --- /dev/null +++ b/go/examples/landmarker/main.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "image" + "image/jpeg" + "log" + "os" + "os/user" + "path/filepath" + "strings" + + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face/detecter" + facedrawer "github.com/bububa/openvision/go/face/drawer" + "github.com/bububa/openvision/go/face/landmarker" +) + +func main() { + wd, _ := os.Getwd() + dataPath := cleanPath(wd, "~/go/src/github.com/bububa/openvision/data") + imgPath := filepath.Join(dataPath, "./images") + modelPath := filepath.Join(dataPath, "./models") + d := retinaface(modelPath) + defer d.Destroy() + m := insightface(modelPath) + defer m.Destroy() + extract_keypoints(d, m, imgPath, "4.jpg") +} + +func retinaface(modelPath string) detecter.Detecter { + modelPath = filepath.Join(modelPath, "fd") + d := detecter.NewRetinaFace() + if err := d.LoadModel(modelPath); err != nil { + log.Fatalln(err) + } + return d +} + +func insightface(modelPath string) landmarker.Landmarker { + modelPath = filepath.Join(modelPath, "insightface") + d := landmarker.NewInsightface() + if err := d.LoadModel(modelPath); err != nil { + log.Fatalln(err) + } + return d +} + +func zq(modelPath string) landmarker.Landmarker { + modelPath = filepath.Join(modelPath, "zq") + d := landmarker.NewZq() + if err := d.LoadModel(modelPath); err != nil { + log.Fatalln(err) + } + return d +} + +func extract_keypoints(d detecter.Detecter, m landmarker.Landmarker, imgPath string, filename string) { + inPath := filepath.Join(imgPath, filename) + imgLoaded, err := loadImage(inPath) + if err != nil { + log.Fatalln("load image failed,", err) + } + img := common.NewImage(imgLoaded) + faces, err := d.DetectFace(img) + if err != nil { + log.Fatalln(err) + } + drawer := facedrawer.New( + facedrawer.WithKeypointColor(common.Red), + facedrawer.WithKeypointRadius(1), + facedrawer.WithKeypointStrokeWidth(0), + ) + var keypoints []common.Point + for _, face := range faces { + rect := face.Rect + points, err := m.ExtractKeypoints(img, rect) + if err != nil { + log.Fatalln(err) + } + keypoints = append(keypoints, points...) + } + out := drawer.DrawLandmark(imgLoaded, keypoints) + outPath := filepath.Join(imgPath, "./results", filename) + + if err := saveImage(out, outPath); err != nil { + log.Fatalln(err) + } + +} + +func loadImage(filePath string) (image.Image, error) { + fn, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer fn.Close() + img, _, err := image.Decode(fn) + if err != nil { + return nil, err + } + return img, nil +} + +func saveImage(img image.Image, filePath string) error { + buf := new(bytes.Buffer) + if err := jpeg.Encode(buf, img, nil); err != nil { + return err + } + fn, err := os.Create(filePath) + if err != nil { + return err + } + defer fn.Close() + fn.Write(buf.Bytes()) + return nil +} + +func cleanPath(wd string, path string) string { + usr, _ := user.Current() + dir := usr.HomeDir + if path == "~" { + return dir + } else if strings.HasPrefix(path, "~/") { + return filepath.Join(dir, path[2:]) + } + return filepath.Join(wd, path) +} diff --git a/go/examples/recognizer/main.go b/go/examples/recognizer/main.go new file mode 100644 index 0000000..c87923f --- /dev/null +++ b/go/examples/recognizer/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "bytes" + "image" + "image/jpeg" + "log" + "os" + "os/user" + "path/filepath" + "strings" + + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face/detecter" + "github.com/bububa/openvision/go/face/recognizer" +) + +func main() { + wd, _ := os.Getwd() + dataPath := cleanPath(wd, "~/go/src/github.com/bububa/openvision/data") + imgPath := filepath.Join(dataPath, "./images") + modelPath := filepath.Join(dataPath, "./models") + d := retinaface(modelPath) + defer d.Destroy() + m := mobilefacenet(modelPath) + defer m.Destroy() + extract_features(d, m, imgPath, "4.jpg") +} + +func retinaface(modelPath string) detecter.Detecter { + modelPath = filepath.Join(modelPath, "fd") + d := detecter.NewRetinaFace() + if err := d.LoadModel(modelPath); err != nil { + log.Fatalln(err) + } + return d +} + +func mobilefacenet(modelPath string) recognizer.Recognizer { + modelPath = filepath.Join(modelPath, "mobilefacenet") + d := recognizer.NewMobilefacenet() + if err := d.LoadModel(modelPath); err != nil { + log.Fatalln(err) + } + return d +} + +func extract_features(d detecter.Detecter, r recognizer.Recognizer, imgPath string, filename string) { + inPath := filepath.Join(imgPath, filename) + imgLoaded, err := loadImage(inPath) + if err != nil { + log.Fatalln("load image failed,", err) + } + img := common.NewImage(imgLoaded) + faces, err := d.DetectFace(img) + if err != nil { + log.Fatalln(err) + } + for _, face := range faces { + rect := face.Rect + log.Printf("x:%f, y:%f, w:%f, h:%f\n", rect.X*img.WidthF64(), rect.Y*img.HeightF64(), rect.Width*img.WidthF64(), rect.Height*img.HeightF64()) + features, err := r.ExtractFeatures(img, rect) + if err != nil { + log.Fatalln(err) + } + log.Println(len(features)) + log.Printf("feature: %+v\n", features) + } +} + +func loadImage(filePath string) (image.Image, error) { + fn, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer fn.Close() + img, _, err := image.Decode(fn) + if err != nil { + return nil, err + } + return img, nil +} + +func saveImage(img image.Image, filePath string) error { + buf := new(bytes.Buffer) + if err := jpeg.Encode(buf, img, nil); err != nil { + return err + } + fn, err := os.Create(filePath) + if err != nil { + return err + } + defer fn.Close() + fn.Write(buf.Bytes()) + return nil +} + +func cleanPath(wd string, path string) string { + usr, _ := user.Current() + dir := usr.HomeDir + if path == "~" { + return dir + } else if strings.HasPrefix(path, "~/") { + return filepath.Join(dir, path[2:]) + } + return filepath.Join(wd, path) +} diff --git a/go/examples/tracker/main.go b/go/examples/tracker/main.go new file mode 100644 index 0000000..1bc0585 --- /dev/null +++ b/go/examples/tracker/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "bytes" + "fmt" + "image" + "image/jpeg" + "log" + "os" + "os/user" + "path/filepath" + "strings" + + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face" + "github.com/bububa/openvision/go/face/detecter" + facedrawer "github.com/bububa/openvision/go/face/drawer" + "github.com/bububa/openvision/go/face/tracker" +) + +func main() { + wd, _ := os.Getwd() + dataPath := cleanPath(wd, "~/go/src/github.com/bububa/openvision/data") + imgPath := filepath.Join(dataPath, "./images") + modelPath := filepath.Join(dataPath, "./models") + d := retinaface(modelPath) + defer d.Destroy() + t := tracker.NewTracker() + defer t.Destroy() + track(d, t, imgPath, "4.jpg") +} + +func retinaface(modelPath string) detecter.Detecter { + modelPath = filepath.Join(modelPath, "fd") + d := detecter.NewRetinaFace() + if err := d.LoadModel(modelPath); err != nil { + log.Fatalln(err) + } + return d +} + +func track(d detecter.Detecter, t *tracker.Tracker, imgPath string, filename string) { + inPath := filepath.Join(imgPath, filename) + imgLoaded, err := loadImage(inPath) + if err != nil { + log.Fatalln("load image failed,", err) + } + img := common.NewImage(imgLoaded) + faces, err := d.DetectFace(img) + if err != nil { + log.Fatalln(err) + } + trackedFaces, err := t.Track(img, faces) + if err != nil { + log.Fatalln(err) + } + infos := make([]face.FaceInfo, 0, len(trackedFaces)) + for _, f := range trackedFaces { + infos = append(infos, f.Face) + } + drawer := facedrawer.New() + out := drawer.Draw(img, infos) + outPath := filepath.Join(imgPath, "./results", fmt.Sprintf("track-%s", filename)) + + if err := saveImage(out, outPath); err != nil { + log.Fatalln(err) + } + +} + +func loadImage(filePath string) (image.Image, error) { + fn, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer fn.Close() + img, _, err := image.Decode(fn) + if err != nil { + return nil, err + } + return img, nil +} + +func saveImage(img image.Image, filePath string) error { + buf := new(bytes.Buffer) + if err := jpeg.Encode(buf, img, nil); err != nil { + return err + } + fn, err := os.Create(filePath) + if err != nil { + return err + } + defer fn.Close() + fn.Write(buf.Bytes()) + return nil +} + +func cleanPath(wd string, path string) string { + usr, _ := user.Current() + dir := usr.HomeDir + if path == "~" { + return dir + } else if strings.HasPrefix(path, "~/") { + return filepath.Join(dir, path[2:]) + } + return filepath.Join(wd, path) +} diff --git a/go/face/cgo.go b/go/face/cgo.go new file mode 100644 index 0000000..22b4849 --- /dev/null +++ b/go/face/cgo.go @@ -0,0 +1,9 @@ +package face + +/* +#cgo CXXFLAGS: --std=c++11 -fopenmp +#cgo CPPFLAGS: -I ${SRCDIR}/../../include -I /usr/local/include +#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision +#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../lib +*/ +import "C" diff --git a/go/face/detecter/anticonv.go b/go/face/detecter/anticonv.go new file mode 100644 index 0000000..d8b85bd --- /dev/null +++ b/go/face/detecter/anticonv.go @@ -0,0 +1,44 @@ +package detecter + +/* +#include +#include +#include "openvision/face/detecter.h" +*/ +import "C" +import ( + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face" +) + +// Anticonv represents anticonv detecter +type Anticonv struct { + d C.IDetecter +} + +// NewAnticonv returns a new Anticonv +func NewAnticonv() *Anticonv { + return &Anticonv{ + d: C.new_anticonv(), + } +} + +// Destroy free detecter +func (d *Anticonv) Destroy() { + Destroy(d) +} + +// Handler returns C.IDetecter +func (d *Anticonv) Handler() C.IDetecter { + return d.d +} + +// LoadModel load model for detecter +func (d *Anticonv) LoadModel(modelPath string) error { + return LoadModel(d, modelPath) +} + +// DetectFace implement Detecter interface +func (d *Anticonv) DetectFace(img *common.Image) ([]face.FaceInfo, error) { + return DetectFace(d, img) +} diff --git a/go/face/detecter/centerface.go b/go/face/detecter/centerface.go new file mode 100644 index 0000000..2aaed00 --- /dev/null +++ b/go/face/detecter/centerface.go @@ -0,0 +1,44 @@ +package detecter + +/* +#include +#include +#include "openvision/face/detecter.h" +*/ +import "C" +import ( + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face" +) + +// Centerface represents centerface detecter +type Centerface struct { + d C.IDetecter +} + +// NewCenterface returns a new Centerface +func NewCenterface() *Centerface { + return &Centerface{ + d: C.new_centerface(), + } +} + +// Destroy free detecter +func (d *Centerface) Destroy() { + Destroy(d) +} + +// Handler returns C.IDetecter +func (d *Centerface) Handler() C.IDetecter { + return d.d +} + +// LoadModel load model for detecter +func (d *Centerface) LoadModel(modelPath string) error { + return LoadModel(d, modelPath) +} + +// DetectFace implement Detecter interface +func (d *Centerface) DetectFace(img *common.Image) ([]face.FaceInfo, error) { + return DetectFace(d, img) +} diff --git a/go/face/detecter/cgo.go b/go/face/detecter/cgo.go new file mode 100644 index 0000000..ea95b10 --- /dev/null +++ b/go/face/detecter/cgo.go @@ -0,0 +1,9 @@ +package detecter + +/* +#cgo CXXFLAGS: --std=c++11 -fopenmp +#cgo CPPFLAGS: -I ${SRCDIR}/../../../include -I /usr/local/include +#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision +#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../../lib +*/ +import "C" diff --git a/go/face/detecter/detecter.go b/go/face/detecter/detecter.go new file mode 100644 index 0000000..5f0ddee --- /dev/null +++ b/go/face/detecter/detecter.go @@ -0,0 +1,55 @@ +package detecter + +/* +#include +#include +#include "openvision/common/common.h" +#include "openvision/face/detecter.h" +*/ +import "C" +import ( + "unsafe" + + openvision "github.com/bububa/openvision/go" + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face" +) + +// Detecter represents deteter interface +type Detecter interface { + Handler() C.IDetecter + LoadModel(modelPath string) error + DetectFace(img *common.Image) ([]face.FaceInfo, error) + Destroy() +} + +// LoadModel load detecter model +func LoadModel(d Detecter, modelPath string) error { + cpath := C.CString(modelPath) + defer C.free(unsafe.Pointer(cpath)) + retCode := C.detecter_load_model(d.Handler(), cpath) + if retCode != 0 { + return openvision.LoadModelError(int(retCode)) + } + return nil +} + +// Destroy a detecter +func Destroy(d Detecter) { + C.destroy_detecter(d.Handler()) +} + +// DetectFace detect face useing detecter +func DetectFace(d Detecter, img *common.Image) ([]face.FaceInfo, error) { + imgWidth := img.WidthF64() + imgHeight := img.HeightF64() + data := img.Bytes() + CFaces := face.NewCFaceInfoVector() + defer face.FreeCFaceInfoVector(CFaces) + errCode := C.detect_face(d.Handler(), (*C.uchar)(unsafe.Pointer(&data[0])), C.int(imgWidth), C.int(imgHeight), (*C.FaceInfoVector)(unsafe.Pointer(CFaces))) + if errCode != 0 { + return nil, openvision.DetectFaceError(int(errCode)) + } + faces := face.GoFaceInfoVector(CFaces, imgWidth, imgHeight) + return faces, nil +} diff --git a/go/face/detecter/doc.go b/go/face/detecter/doc.go new file mode 100644 index 0000000..05052ba --- /dev/null +++ b/go/face/detecter/doc.go @@ -0,0 +1,2 @@ +// Package detecter face detecter +package detecter diff --git a/go/face/detecter/mtcnn.go b/go/face/detecter/mtcnn.go new file mode 100644 index 0000000..4445a25 --- /dev/null +++ b/go/face/detecter/mtcnn.go @@ -0,0 +1,44 @@ +package detecter + +/* +#include +#include +#include "openvision/face/detecter.h" +*/ +import "C" +import ( + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face" +) + +// Mtcnn represents mtcnn detecter +type Mtcnn struct { + d C.IDetecter +} + +// NewMtcnn returns a new Mtcnn +func NewMtcnn() *Mtcnn { + return &Mtcnn{ + d: C.new_mtcnn(), + } +} + +// Destroy free detecter +func (d *Mtcnn) Destroy() { + Destroy(d) +} + +// Handler returns C.IDetecter +func (d *Mtcnn) Handler() C.IDetecter { + return d.d +} + +// LoadModel implement Detecter interface +func (d *Mtcnn) LoadModel(modelPath string) error { + return LoadModel(d, modelPath) +} + +// DetectFace implement Detecter interface +func (d *Mtcnn) DetectFace(img *common.Image) ([]face.FaceInfo, error) { + return DetectFace(d, img) +} diff --git a/go/face/detecter/retinaface.go b/go/face/detecter/retinaface.go new file mode 100644 index 0000000..35b299f --- /dev/null +++ b/go/face/detecter/retinaface.go @@ -0,0 +1,41 @@ +package detecter + +/* +#include +#include +#include "openvision/face/detecter.h" +*/ +import "C" +import ( + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face" +) + +// RetinaFace represents retinaface detecter +type RetinaFace struct { + d C.IDetecter +} + +// NewRetinaFace returns a new RetinaFace +func NewRetinaFace() *RetinaFace { + return &RetinaFace{ + d: C.new_retinaface(), + } +} + +// Destroy free detecter +func (d *RetinaFace) Destroy() { + C.destroy_detecter(d.d) +} + +func (d *RetinaFace) Handler() C.IDetecter { + return d.d +} + +func (d *RetinaFace) LoadModel(modelPath string) error { + return LoadModel(d, modelPath) +} + +func (d *RetinaFace) DetectFace(img *common.Image) ([]face.FaceInfo, error) { + return DetectFace(d, img) +} diff --git a/go/face/doc.go b/go/face/doc.go new file mode 100644 index 0000000..e18c86a --- /dev/null +++ b/go/face/doc.go @@ -0,0 +1,2 @@ +// Package face include face detecter/landmarker/reconginzier +package face diff --git a/go/face/drawer/const.go b/go/face/drawer/const.go new file mode 100644 index 0000000..a646261 --- /dev/null +++ b/go/face/drawer/const.go @@ -0,0 +1,20 @@ +package face + +import ( + "github.com/bububa/openvision/go/common" +) + +const ( + // DefaultBorderColor default drawer border color + DefaultBorderColor = common.Green + // DefaultKeypointColor default drawer keypoint color + DefaultKeypointColor = common.Pink + // DefaultBorderStrokeWidth default drawer border stroke width + DefaultBorderStrokeWidth = 3 + // DefaultKeypointRadius default drawer keypoint radius + DefaultKeypointRadius = 2 + // DefaultKeypointStrokeWidth default drawer keypoint stroke width + DefaultKeypointStrokeWidth = 2 + // DefaultInvalidBorderColor default drawer invalid border color + DefaultInvalidBorderColor = common.Red +) diff --git a/go/face/drawer/drawer.go b/go/face/drawer/drawer.go new file mode 100644 index 0000000..cbe2683 --- /dev/null +++ b/go/face/drawer/drawer.go @@ -0,0 +1,86 @@ +package face + +import ( + "image" + + "github.com/llgcode/draw2d/draw2dimg" + + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face" +) + +// Drawer represents a face drawer +type Drawer struct { + // BorderColor represents face rect border color + BorderColor string + // KeypointColor represents keypoint color + KeypointColor string + // BorderStrokeWidth represents face rect stroke width + BorderStrokeWidth float64 + // KeypointRadius represents keypoints circle radius + KeypointRadius float64 + // KeypointStrokeWidth represents keypoints stroke width + KeypointStrokeWidth float64 + // MaskColor represents border color which mask is true + MaskColor string + // InvalidBorderColor + InvalidBorderColor string +} + +// New returns a new Drawer +func New(options ...Option) *Drawer { + d := &Drawer{ + BorderColor: DefaultBorderColor, + BorderStrokeWidth: DefaultBorderStrokeWidth, + KeypointColor: DefaultKeypointColor, + KeypointStrokeWidth: DefaultKeypointStrokeWidth, + KeypointRadius: DefaultKeypointRadius, + InvalidBorderColor: DefaultInvalidBorderColor, + MaskColor: DefaultBorderColor, + } + for _, opt := range options { + opt.apply(d) + } + return d +} + +// Draw draw faces +func (d *Drawer) Draw(img image.Image, faces []face.FaceInfo) image.Image { + imgW := float64(img.Bounds().Dx()) + imgH := float64(img.Bounds().Dy()) + out := image.NewRGBA(img.Bounds()) + gc := draw2dimg.NewGraphicContext(out) + gc.DrawImage(img) + for _, face := range faces { + // draw rect + rect := common.Rect( + face.Rect.X*imgW, + face.Rect.Y*imgH, + face.Rect.Width*imgW, + face.Rect.Height*imgH, + ) + borderColor := d.BorderColor + if face.Mask { + borderColor = d.MaskColor + } + common.DrawRectangle(gc, rect, borderColor, "", d.BorderStrokeWidth) + // draw keypoints + for _, pt := range face.Keypoints { + common.DrawCircle(gc, common.Pt(pt.X*imgW, pt.Y*imgH), d.KeypointRadius, d.KeypointColor, "", d.KeypointStrokeWidth) + } + } + return out +} + +// DrawLandmark draw landmark +func (d *Drawer) DrawLandmark(img image.Image, points []common.Point) image.Image { + imgW := float64(img.Bounds().Dx()) + imgH := float64(img.Bounds().Dy()) + out := image.NewRGBA(img.Bounds()) + gc := draw2dimg.NewGraphicContext(out) + gc.DrawImage(img) + for _, pt := range points { + common.DrawCircle(gc, common.Pt(pt.X*imgW, pt.Y*imgH), d.KeypointRadius, d.KeypointColor, "", d.KeypointStrokeWidth) + } + return out +} diff --git a/go/face/drawer/option.go b/go/face/drawer/option.go new file mode 100644 index 0000000..6c43fa4 --- /dev/null +++ b/go/face/drawer/option.go @@ -0,0 +1,61 @@ +package face + +// Option represents Drawer option interface +type Option interface { + apply(*Drawer) +} + +type optionFunc func(d *Drawer) + +func (fn optionFunc) apply(d *Drawer) { + fn(d) +} + +// WithBorderColor set Drawer BorderColor +func WithBorderColor(color string) Option { + return optionFunc(func(d *Drawer) { + d.BorderColor = color + }) +} + +// WithKeypointColor set Drawer KeypointColor +func WithKeypointColor(color string) Option { + return optionFunc(func(d *Drawer) { + d.KeypointColor = color + }) +} + +// WithBorderStrokeWidth set Drawer BorderStrokeWidth +func WithBorderStrokeWidth(w float64) Option { + return optionFunc(func(d *Drawer) { + d.BorderStrokeWidth = w + }) +} + +// WithKeypointRadius set Drawer KeypointRadius +func WithKeypointRadius(r float64) Option { + return optionFunc(func(d *Drawer) { + d.KeypointRadius = r + }) +} + +// WithKeypointStrokeWidth set Drawer KeypointStrokeWidth +func WithKeypointStrokeWidth(w float64) Option { + return optionFunc(func(d *Drawer) { + d.KeypointStrokeWidth = w + }) +} + +// WithInvalidBorderColor set Drawer InvalidBorderColor +func WithInvalidBorderColor(color string) Option { + return optionFunc(func(d *Drawer) { + d.InvalidBorderColor = color + }) +} + +// WithMaskColor set Drawer MaskColor +func WithMaskColor(color string) Option { + return optionFunc(func(d *Drawer) { + d.MaskColor = color + }) +} diff --git a/go/face/face_info.go b/go/face/face_info.go new file mode 100644 index 0000000..a7f11e0 --- /dev/null +++ b/go/face/face_info.go @@ -0,0 +1,101 @@ +package face + +/* +#include +#include +#include "openvision/common/common.h" +*/ +import "C" +import ( + "unsafe" + + "github.com/bububa/openvision/go/common" +) + +// FaceInfo represents detected face info +type FaceInfo struct { + // Rect face location + Rect common.Rectangle + // Score detected score + Score float32 + // Keypoints . + Keypoints [5]common.Point + // Mask has mask or not + Mask bool +} + +// GoFaceInfo convert c FaceInfo to go type +func GoFaceInfo(cInfo *C.FaceInfo, w float64, h float64) FaceInfo { + info := FaceInfo{ + Score: float32(cInfo.score_), + Mask: bool(cInfo.mask_), + Rect: common.Rect( + float64(cInfo.location_.x)/w, + float64(cInfo.location_.y)/h, + float64(cInfo.location_.width)/w, + float64(cInfo.location_.height)/h, + ), + } + for i := 0; i < 5; i++ { + info.Keypoints[i] = common.Pt( + float64(cInfo.keypoints_[i])/w, + float64(cInfo.keypoints_[i+5])/h, + ) + } + return info +} + +// CFaceInfo convert FaceInfo to C.FaceInfo +func (f FaceInfo) CFaceInfo() *C.FaceInfo { + ret := (*C.FaceInfo)(C.malloc(C.sizeof_FaceInfo)) + ret.score_ = C.float(f.Score) + ret.mask_ = C.bool(f.Mask) + ret.location_ = C.Rect{ + C.int(f.Rect.X), + C.int(f.Rect.Y), + C.int(f.Rect.Width), + C.int(f.Rect.Height), + } + for i := 0; i < 5; i++ { + ret.keypoints_[i] = C.float(f.Keypoints[i].X) + ret.keypoints_[i+5] = C.float(f.Keypoints[i].Y) + } + return ret +} + +// NewCFaceInfoVector returns C.FaceInfoVector pointer +func NewCFaceInfoVector() *C.FaceInfoVector { + return (*C.FaceInfoVector)(C.malloc(C.sizeof_FaceInfo)) +} + +// GoFaceInfoVector conver c FaceInfoVector to go FaceInfo slice +func GoFaceInfoVector(cVector *C.FaceInfoVector, w float64, h float64) []FaceInfo { + l := int(cVector.length) + ret := make([]FaceInfo, 0, l) + ptr := unsafe.Pointer(cVector.faces) + for i := 0; i < l; i++ { + cFace := (*C.FaceInfo)(unsafe.Pointer(uintptr(ptr) + uintptr(C.sizeof_FaceInfo*C.int(i)))) + ret = append(ret, GoFaceInfo(cFace, w, h)) + } + return ret +} + +// FreeCFaceInfoVector release CFaceInfoVector memory +func FreeCFaceInfoVector(faces *C.FaceInfoVector) { + C.FreeFaceInfoVector(faces) + C.free(unsafe.Pointer(faces)) +} + +// NewCFaceInfoVectorFromFaces returns C.FaceInfoVector pointer +func NewCFaceInfoVectorFromFaces(faces []FaceInfo) *C.FaceInfoVector { + l := len(faces) + vec := (*C.FaceInfoVector)(C.malloc(C.sizeof_FaceInfoVector)) + C.NewFaceInfoVector(vec, C.int(l)) + p := (*[1 << 30]C.FaceInfo)(unsafe.Pointer(vec.faces))[:l:l] + for i := 0; i < l; i++ { + face := faces[i].CFaceInfo() + defer C.free(unsafe.Pointer(face)) + p[i] = *face + } + return vec +} diff --git a/go/face/landmarker/cgo.go b/go/face/landmarker/cgo.go new file mode 100644 index 0000000..eb3dc0b --- /dev/null +++ b/go/face/landmarker/cgo.go @@ -0,0 +1,9 @@ +package landmarker + +/* +#cgo CXXFLAGS: --std=c++11 -fopenmp +#cgo CPPFLAGS: -I ${SRCDIR}/../../../include -I /usr/local/include +#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision +#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../../lib +*/ +import "C" diff --git a/go/face/landmarker/doc.go b/go/face/landmarker/doc.go new file mode 100644 index 0000000..dc0d2a0 --- /dev/null +++ b/go/face/landmarker/doc.go @@ -0,0 +1,2 @@ +// Package landmarker include landmarker instances +package landmarker diff --git a/go/face/landmarker/insightface.go b/go/face/landmarker/insightface.go new file mode 100644 index 0000000..9a42cbe --- /dev/null +++ b/go/face/landmarker/insightface.go @@ -0,0 +1,41 @@ +package landmarker + +/* +#include +#include +#include "openvision/face/landmarker.h" +*/ +import "C" +import "github.com/bububa/openvision/go/common" + +// Insightface represents Insightface landmarker +type Insightface struct { + d C.ILandmarker +} + +// NewInsightface returns a new Insightface landmarker +func NewInsightface() *Insightface { + return &Insightface{ + d: C.new_insightface(), + } +} + +// Handler returns C.ILandmarker +func (d *Insightface) Handler() C.ILandmarker { + return d.d +} + +// LoadModel implement Landmarker interface +func (d *Insightface) LoadModel(modelPath string) error { + return LoadModel(d, modelPath) +} + +// Destroy implement Landmarker interface +func (d *Insightface) Destroy() { + Destroy(d) +} + +// ExtractKeypoints implement Landmarker interface +func (d *Insightface) ExtractKeypoints(img *common.Image, faceRect common.Rectangle) ([]common.Point, error) { + return ExtractKeypoints(d, img, faceRect) +} diff --git a/go/face/landmarker/landmarker.go b/go/face/landmarker/landmarker.go new file mode 100644 index 0000000..288b942 --- /dev/null +++ b/go/face/landmarker/landmarker.go @@ -0,0 +1,66 @@ +package landmarker + +/* +#include +#include +#include "openvision/common/common.h" +#include "openvision/face/landmarker.h" +*/ +import "C" +import ( + "unsafe" + + openvision "github.com/bububa/openvision/go" + "github.com/bububa/openvision/go/common" +) + +// Landmarker represents landmarker interface +type Landmarker interface { + Handler() C.ILandmarker + LoadModel(modelPath string) error + ExtractKeypoints(img *common.Image, face common.Rectangle) ([]common.Point, error) + Destroy() +} + +// LoadModel load landmarker model +func LoadModel(d Landmarker, modelPath string) error { + cpath := C.CString(modelPath) + defer C.free(unsafe.Pointer(cpath)) + retCode := C.landmarker_load_model(d.Handler(), cpath) + if retCode != 0 { + return openvision.LoadModelError(int(retCode)) + } + return nil +} + +// Destroy a landmarker +func Destroy(d Landmarker) { + C.destroy_landmarker(d.Handler()) +} + +// ExtractKeypoints extract keypoints using landmarker +func ExtractKeypoints(d Landmarker, img *common.Image, faceRect common.Rectangle) ([]common.Point, error) { + imgWidth := img.WidthF64() + imgHeight := img.HeightF64() + faceRect.X *= imgWidth + faceRect.Y *= imgHeight + faceRect.Width *= imgWidth + faceRect.Height *= imgHeight + data := img.Bytes() + CPoints := common.NewCPoint2fVector() + defer common.FreeCPoint2fVector(CPoints) + CRect := faceRect.CRect() + errCode := C.extract_keypoints( + d.Handler(), + (*C.uchar)(unsafe.Pointer(&data[0])), + C.int(imgWidth), C.int(imgHeight), + (*C.Rect)(unsafe.Pointer(CRect)), + (*C.Point2fVector)(unsafe.Pointer(CPoints)), + ) + C.free(unsafe.Pointer(CRect)) + if errCode != 0 { + return nil, openvision.FaceLandmarkError(int(errCode)) + } + points := common.GoPoint2fVector(CPoints, imgWidth, imgHeight) + return points, nil +} diff --git a/go/face/landmarker/zq.go b/go/face/landmarker/zq.go new file mode 100644 index 0000000..5aa35b6 --- /dev/null +++ b/go/face/landmarker/zq.go @@ -0,0 +1,41 @@ +package landmarker + +/* +#include +#include +#include "openvision/face/landmarker.h" +*/ +import "C" +import "github.com/bububa/openvision/go/common" + +// Zq represents Zq landmarker +type Zq struct { + d C.ILandmarker +} + +// NewZq returns a new Zq landmarker +func NewZq() *Zq { + return &Zq{ + d: C.new_zq(), + } +} + +// Handler returns C.ILandmarker +func (d *Zq) Handler() C.ILandmarker { + return d.d +} + +// LoadModel implement Landmarker interface +func (d *Zq) LoadModel(modelPath string) error { + return LoadModel(d, modelPath) +} + +// Destroy implement Landmarker interface +func (d *Zq) Destroy() { + Destroy(d) +} + +// ExtractKeypoints implement Landmarker interface +func (d *Zq) ExtractKeypoints(img *common.Image, faceRect common.Rectangle) ([]common.Point, error) { + return ExtractKeypoints(d, img, faceRect) +} diff --git a/go/face/recognizer/cgo.go b/go/face/recognizer/cgo.go new file mode 100644 index 0000000..619d81d --- /dev/null +++ b/go/face/recognizer/cgo.go @@ -0,0 +1,9 @@ +package recognizer + +/* +#cgo CXXFLAGS: --std=c++11 -fopenmp +#cgo CPPFLAGS: -I ${SRCDIR}/../../../include -I /usr/local/include +#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision +#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../../lib +*/ +import "C" diff --git a/go/face/recognizer/doc.go b/go/face/recognizer/doc.go new file mode 100644 index 0000000..0316175 --- /dev/null +++ b/go/face/recognizer/doc.go @@ -0,0 +1,2 @@ +// Package recognizer include feature extractor +package recognizer diff --git a/go/face/recognizer/mobilefacenet.go b/go/face/recognizer/mobilefacenet.go new file mode 100644 index 0000000..71a14a2 --- /dev/null +++ b/go/face/recognizer/mobilefacenet.go @@ -0,0 +1,41 @@ +package recognizer + +/* +#include +#include +#include "openvision/face/recognizer.h" +*/ +import "C" +import "github.com/bububa/openvision/go/common" + +// Mobilefacenet represents Mobilefacenet recognizer +type Mobilefacenet struct { + d C.IRecognizer +} + +// NewMobilefacenet returns a new Mobilefacenet recognizer +func NewMobilefacenet() *Mobilefacenet { + return &Mobilefacenet{ + d: C.new_mobilefacenet(), + } +} + +// Handler returns C.IRecognizer +func (d *Mobilefacenet) Handler() C.IRecognizer { + return d.d +} + +// LoadModel implement Recognizer interface +func (d *Mobilefacenet) LoadModel(modelPath string) error { + return LoadModel(d, modelPath) +} + +// Destroy implement Recognizer interface +func (d *Mobilefacenet) Destroy() { + Destroy(d) +} + +// ExtractFeatures implement Recognizer interface +func (d *Mobilefacenet) ExtractFeatures(img *common.Image, faceRect common.Rectangle) ([]float64, error) { + return ExtractFeatures(d, img, faceRect) +} diff --git a/go/face/recognizer/recognizer.go b/go/face/recognizer/recognizer.go new file mode 100644 index 0000000..684b392 --- /dev/null +++ b/go/face/recognizer/recognizer.go @@ -0,0 +1,66 @@ +package recognizer + +/* +#include +#include +#include "openvision/common/common.h" +#include "openvision/face/recognizer.h" +*/ +import "C" +import ( + "unsafe" + + openvision "github.com/bububa/openvision/go" + "github.com/bububa/openvision/go/common" +) + +// Recognizer represents Recognizer interface +type Recognizer interface { + Handler() C.IRecognizer + LoadModel(modelPath string) error + ExtractFeatures(img *common.Image, face common.Rectangle) ([]float64, error) + Destroy() +} + +// LoadModel load recognizer model +func LoadModel(r Recognizer, modelPath string) error { + cpath := C.CString(modelPath) + defer C.free(unsafe.Pointer(cpath)) + retCode := C.recognizer_load_model(r.Handler(), cpath) + if retCode != 0 { + return openvision.LoadModelError(int(retCode)) + } + return nil +} + +// Destroy a recognizer +func Destroy(r Recognizer) { + C.destroy_recognizer(r.Handler()) +} + +// ExtractFeatures extract features using recognizer +func ExtractFeatures(r Recognizer, img *common.Image, faceRect common.Rectangle) ([]float64, error) { + imgWidth := img.WidthF64() + imgHeight := img.HeightF64() + faceRect.X *= imgWidth + faceRect.Y *= imgHeight + faceRect.Width *= imgWidth + faceRect.Height *= imgHeight + data := img.Bytes() + CFeatures := common.NewCFloatVector() + defer common.FreeCFloatVector(CFeatures) + CRect := faceRect.CRect() + errCode := C.extract_feature( + r.Handler(), + (*C.uchar)(unsafe.Pointer(&data[0])), + C.int(imgWidth), C.int(imgHeight), + (*C.Rect)(unsafe.Pointer(CRect)), + (*C.FloatVector)(unsafe.Pointer(CFeatures)), + ) + C.free(unsafe.Pointer(CRect)) + if errCode != 0 { + return nil, openvision.RecognizeFaceError(int(errCode)) + } + features := common.GoFloatVector(CFeatures) + return features, nil +} diff --git a/go/face/tracked_face_info.go b/go/face/tracked_face_info.go new file mode 100644 index 0000000..cdda8bf --- /dev/null +++ b/go/face/tracked_face_info.go @@ -0,0 +1,48 @@ +package face + +/* +#include +#include +#include "openvision/common/common.h" +*/ +import "C" +import ( + "unsafe" +) + +// TrackedFaceInfo represents detected tracked face info +type TrackedFaceInfo struct { + Face FaceInfo + IOUScore float64 +} + +// GoTrackedFaceInfo convert c TrackedFaceInfo to go type +func GoTrackedFaceInfo(cInfo *C.TrackedFaceInfo, w float64, h float64) TrackedFaceInfo { + return TrackedFaceInfo{ + Face: GoFaceInfo(&cInfo.face_info_, w, h), + IOUScore: float64(cInfo.iou_score_), + } +} + +// NewCTrackedFaceInfoVector returns C.TrackedFaceInfoVector pointer +func NewCTrackedFaceInfoVector() *C.TrackedFaceInfoVector { + return (*C.TrackedFaceInfoVector)(C.malloc(C.sizeof_TrackedFaceInfo)) +} + +// GoTrackedFaceInfoVector conver c TrackedFaceInfoVector to go TrackedFaceInfo slice +func GoTrackedFaceInfoVector(cVector *C.TrackedFaceInfoVector, w float64, h float64) []TrackedFaceInfo { + l := int(cVector.length) + ret := make([]TrackedFaceInfo, 0, l) + ptr := unsafe.Pointer(cVector.faces) + for i := 0; i < l; i++ { + cFace := (*C.TrackedFaceInfo)(unsafe.Pointer(uintptr(ptr) + uintptr(C.sizeof_TrackedFaceInfo*C.int(i)))) + ret = append(ret, GoTrackedFaceInfo(cFace, w, h)) + } + return ret +} + +// FreeCTrackedFaceInfoVector release CTrackedFaceInfoVector memory +func FreeCTrackedFaceInfoVector(faces *C.TrackedFaceInfoVector) { + C.FreeTrackedFaceInfoVector(faces) + C.free(unsafe.Pointer(faces)) +} diff --git a/go/face/tracker/cgo.go b/go/face/tracker/cgo.go new file mode 100644 index 0000000..8c464cb --- /dev/null +++ b/go/face/tracker/cgo.go @@ -0,0 +1,9 @@ +package tracker + +/* +#cgo CXXFLAGS: --std=c++11 -fopenmp +#cgo CPPFLAGS: -I ${SRCDIR}/../../../include -I /usr/local/include +#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision +#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../../lib +*/ +import "C" diff --git a/go/face/tracker/doc.go b/go/face/tracker/doc.go new file mode 100644 index 0000000..b4f755e --- /dev/null +++ b/go/face/tracker/doc.go @@ -0,0 +1,2 @@ +// Package tracker defines face Tracker +package tracker diff --git a/go/face/tracker/tracker.go b/go/face/tracker/tracker.go new file mode 100644 index 0000000..5848915 --- /dev/null +++ b/go/face/tracker/tracker.go @@ -0,0 +1,66 @@ +package tracker + +/* +#include +#include +#include "openvision/common/common.h" +#include "openvision/face/tracker.h" +*/ +import "C" +import ( + "unsafe" + + openvision "github.com/bububa/openvision/go" + "github.com/bububa/openvision/go/common" + "github.com/bububa/openvision/go/face" +) + +// Tracker represents Tracker +type Tracker struct { + d C.ITracker +} + +// NewTracker returns a new Tracker +func NewTracker() *Tracker { + return &Tracker{ + d: C.new_tracker(), + } +} + +// Destroy destroy C.ITracker +func (t *Tracker) Destroy() { + C.destroy_tracker(t.d) +} + +// Track track faces +func (t *Tracker) Track(img *common.Image, faces []face.FaceInfo) ([]face.TrackedFaceInfo, error) { + imgWidth := img.WidthF64() + imgHeight := img.HeightF64() + l := len(faces) + currFaces := make([]face.FaceInfo, 0, l) + for _, f := range faces { + f.Rect.X *= imgWidth + f.Rect.Y *= imgHeight + f.Rect.Width *= imgWidth + f.Rect.Height *= imgHeight + for i := 0; i < 5; i++ { + f.Keypoints[i].X *= imgWidth + f.Keypoints[i].Y *= imgHeight + } + currFaces = append(currFaces, f) + } + CCurrFaces := face.NewCFaceInfoVectorFromFaces(currFaces) + defer face.FreeCFaceInfoVector(CCurrFaces) + CTrackedFaces := face.NewCTrackedFaceInfoVector() + defer face.FreeCTrackedFaceInfoVector(CTrackedFaces) + errCode := C.track( + t.d, + (*C.FaceInfoVector)(unsafe.Pointer(CCurrFaces)), + (*C.TrackedFaceInfoVector)(unsafe.Pointer(CTrackedFaces)), + ) + if errCode != 0 { + return nil, openvision.TrackFaceError(int(errCode)) + } + trackedFaces := face.GoTrackedFaceInfoVector(CTrackedFaces, imgWidth, imgHeight) + return trackedFaces, nil +} diff --git a/go/go.sum b/go/go.sum new file mode 100644 index 0000000..88b247b --- /dev/null +++ b/go/go.sum @@ -0,0 +1,11 @@ +github.com/go-gl/gl v0.0.0-20180407155706-68e253793080/go.mod h1:482civXOzJJCPzJ4ZOX/pwvXBWSnzD4OKMdH4ClKGbk= +github.com/go-gl/glfw v0.0.0-20180426074136-46a8d530c326/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/llgcode/draw2d v0.0.0-20210904075650-80aa0a2a901d h1:4/ycg+VrwjGurTqiHv2xM/h6Qm81qSra+KbfT4FH2FA= +github.com/llgcode/draw2d v0.0.0-20210904075650-80aa0a2a901d/go.mod h1:mVa0dA29Db2S4LVqDYLlsePDzRJLDfdhVZiI15uY0FA= +github.com/llgcode/ps v0.0.0-20150911083025-f1443b32eedb h1:61ndUreYSlWFeCY44JxDDkngVoI7/1MVhEl98Nm0KOk= +github.com/llgcode/ps v0.0.0-20150911083025-f1443b32eedb/go.mod h1:1l8ky+Ew27CMX29uG+a2hNOKpeNYEQjjtiALiBlFQbY= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 h1:00VmoueYNlNz/aHIilyyQz/MHSqGoWJzpFv/HW8xpzI= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= diff --git a/include/.gitignore b/include/.gitignore new file mode 100644 index 0000000..94548af --- /dev/null +++ b/include/.gitignore @@ -0,0 +1,3 @@ +* +*/ +!.gitignore diff --git a/lib/.gitignore b/lib/.gitignore new file mode 100644 index 0000000..94548af --- /dev/null +++ b/lib/.gitignore @@ -0,0 +1,3 @@ +* +*/ +!.gitignore diff --git a/ncnn b/ncnn new file mode 160000 index 0000000..e3aa893 --- /dev/null +++ b/ncnn @@ -0,0 +1 @@ +Subproject commit e3aa893dfb310eaea801975e2bf039d85aa0eaf8 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..2e7c4ab --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,63 @@ +file(GLOB_RECURSE SRC_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp +) + +message(${SRC_FILES}) + +# list(APPEND SRCS ${LAYER_ARCH_SRC}) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O2 -fPIC -std=c++11 -fopenmp") +add_library(openvision STATIC ${SRC_FILES}) +target_link_libraries(openvision PUBLIC ncnn) + +if(MIRROR_OPENMP) + find_package(OpenMP) + if(NOT TARGET OpenMP::OpenMP_CXX AND (OpenMP_CXX_FOUND OR OPENMP_FOUND)) + target_compile_options(openvision PRIVATE ${OpenMP_CXX_FLAGS}) + endif() +endif() + +if(MIRROR_OPENMP AND OpenMP_CXX_FOUND) + message("Building with OpenMP") + target_link_libraries(openvision PUBLIC OpenMP::OpenMP_CXX) +endif() + +if(MIRROR_VULKAN) + find_package(Vulkan REQUIRED) + target_link_libraries(openvision PUBLIC Vulkan::Vulkan) +endif() + +target_include_directories(openvision + PUBLIC + $ + $ + $ + + $ + #$ + $ + $ + $ + $ + $ + + $ + $ + $ + + $ + $ + #$ +) + +#install(TARGETS openvision EXPORT openvision ARCHIVE DESTINATION ${LIBRARY_OUTPUT_PATH}) +file(COPY + ${CMAKE_CURRENT_SOURCE_DIR}/common/common.h + DESTINATION ${INCLUDE_OUTPUT_PATH}/openvision/common +) +file(COPY + ${CMAKE_CURRENT_SOURCE_DIR}/face/detecter/detecter.h + ${CMAKE_CURRENT_SOURCE_DIR}/face/landmarker/landmarker.h + ${CMAKE_CURRENT_SOURCE_DIR}/face/recognizer/recognizer.h + ${CMAKE_CURRENT_SOURCE_DIR}/face/tracker/tracker.h + DESTINATION ${INCLUDE_OUTPUT_PATH}/openvision/face +) diff --git a/src/common/common.cpp b/src/common/common.cpp new file mode 100644 index 0000000..2b6482d --- /dev/null +++ b/src/common/common.cpp @@ -0,0 +1,172 @@ +#include "common.h" +#include +#include +#include + +void FreePoint2fVector(Point2fVector* p) { + if (p->points != NULL) { + free(p->points); + p->points = NULL; + } +} + +void FreeRectVector(RectVector *p) { + if (p->rects != NULL) { + free(p->rects); + p->rects = NULL; + } +} + +void FreeFaceInfoVector(FaceInfoVector *p) { + if (p->faces != NULL) { + free(p->faces); + p->faces = NULL; + } +} + +void NewFaceInfoVector(FaceInfoVector *v, int num) { + v->length = num; + v->faces = (FaceInfo*)(malloc(num * sizeof(FaceInfo))); +} + +void FreeTrackedFaceInfoVector(TrackedFaceInfoVector *p) { + if (p->faces != NULL) { + free(p->faces); + p->faces = NULL; + } +} + +void FreeFloatVector(FloatVector *p) { + if (p->values != NULL) { + free(p->values); + p->values = NULL; + } +} + +namespace mirror { + +int RatioAnchors(const Rect & anchor, + const std::vector& ratios, + std::vector* anchors) { + anchors->clear(); + Point center = Point(anchor.x + (anchor.width - 1) * 0.5f, + anchor.y + (anchor.height - 1) * 0.5f); + float anchor_size = anchor.width * anchor.height; +#if defined(_OPENMP) +#pragma omp parallel for num_threads(threads_num) +#endif + for (int i = 0; i < static_cast(ratios.size()); ++i) { + float ratio = ratios.at(i); + float anchor_size_ratio = anchor_size / ratio; + float curr_anchor_width = sqrt(anchor_size_ratio); + float curr_anchor_height = curr_anchor_width * ratio; + float curr_x = center.x - (curr_anchor_width - 1)* 0.5f; + float curr_y = center.y - (curr_anchor_height - 1)* 0.5f; + + Rect curr_anchor = Rect(curr_x, curr_y, + curr_anchor_width - 1, curr_anchor_height - 1); + anchors->push_back(curr_anchor); + } + return 0; +} + +int ScaleAnchors(const std::vector& ratio_anchors, + const std::vector& scales, std::vector* anchors) { + anchors->clear(); +#if defined(_OPENMP) +#pragma omp parallel for num_threads(threads_num) +#endif + for (int i = 0; i < static_cast(ratio_anchors.size()); ++i) { + Rect anchor = ratio_anchors.at(i); + Point2f center = Point2f(anchor.x + anchor.width * 0.5f, + anchor.y + anchor.height * 0.5f); + for (int j = 0; j < static_cast(scales.size()); ++j) { + float scale = scales.at(j); + float curr_width = scale * (anchor.width + 1); + float curr_height = scale * (anchor.height + 1); + float curr_x = center.x - curr_width * 0.5f; + float curr_y = center.y - curr_height * 0.5f; + Rect curr_anchor = Rect(curr_x, curr_y, + curr_width, curr_height); + anchors->push_back(curr_anchor); + } + } + + return 0; +} + +int GenerateAnchors(const int & base_size, + const std::vector& ratios, + const std::vector scales, + std::vector* anchors) { + anchors->clear(); + Rect anchor = Rect(0, 0, base_size, base_size); + std::vector ratio_anchors; + RatioAnchors(anchor, ratios, &ratio_anchors); + ScaleAnchors(ratio_anchors, scales, anchors); + + return 0; +} + +float InterRectArea(const Rect & a, const Rect & b) { + Point left_top = Point(std::max(a.x, b.x), std::max(a.y, b.y)); + Point right_bottom = Point(std::min(a.br().x, b.br().x), std::min(a.br().y, b.br().y)); + Point diff = right_bottom - left_top; + return (std::max(diff.x + 1, 0) * std::max(diff.y + 1, 0)); +} + +int ComputeIOU(const Rect & rect1, + const Rect & rect2, float * iou, + const std::string& type) { + + float inter_area = InterRectArea(rect1, rect2); + if (type == "UNION") { + *iou = inter_area / (rect1.area() + rect2.area() - inter_area); + } + else { + *iou = inter_area / std::min(rect1.area(), rect2.area()); + } + + return 0; +} + +float CalculateSimilarity(const std::vector&feature1, const std::vector& feature2) { + if (feature1.size() != feature2.size()) { + std::cout << "feature size not match." << std::endl; + return 10003; + } + float inner_product = 0.0f; + float feature_norm1 = 0.0f; + float feature_norm2 = 0.0f; +#if defined(_OPENMP) +#pragma omp parallel for num_threads(threads_num) +#endif + for(int i = 0; i < kFaceFeatureDim; ++i) { + inner_product += feature1[i] * feature2[i]; + feature_norm1 += feature1[i] * feature1[i]; + feature_norm2 += feature2[i] * feature2[i]; + } + return inner_product / sqrt(feature_norm1) / sqrt(feature_norm2); +} + +void EnlargeRect(const float& scale, Rect* rect) { + float offset_x = (scale - 1.f) / 2.f * rect->width; + float offset_y = (scale - 1.f) / 2.f * rect->height; + rect->x -= offset_x; + rect->y -= offset_y; + rect->width = scale * rect->width; + rect->height = scale * rect->height; +} + +void RectifyRect(Rect* rect) { + int max_side = std::max(rect->width, rect->height); + int offset_x = (max_side - rect->width) / 2; + int offset_y = (max_side - rect->height) / 2; + + rect->x -= offset_x; + rect->y -= offset_y; + rect->width = max_side; + rect->height = max_side; +} + +} diff --git a/src/common/common.h b/src/common/common.h new file mode 100644 index 0000000..f5bc806 --- /dev/null +++ b/src/common/common.h @@ -0,0 +1,103 @@ +#ifndef _COMMON_C_H_ +#define _COMMON_C_H_ + +#ifdef __cplusplus +#include "common.hpp" +extern "C" { +#endif + +#ifdef __cplusplus +typedef mirror::Size Size; +typedef mirror::Point Point; +typedef mirror::Point2f Point2f; +typedef mirror::Rect Rect; +typedef mirror::FaceInfo FaceInfo; +typedef mirror::TrackedFaceInfo TrackedFaceInfo; +#else + +#define kFaceFeatureDim 128 +#define kFaceNameDim 256 + +// Wrapper for an individual cv::cvSize +typedef struct Size { + int width; + int height; +} Size; + +// Wrapper for an individual cv::cvPoint +typedef struct Point { + int x; + int y; +} Point; + +// Wrapper for an individual cv::Point2f +typedef struct Point2f { + float x; + float y; +} Point2f; + + +// Wrapper for an individual cv::Rect +typedef struct Rect { + int x; + int y; + int width; + int height; +} Rect; + + +typedef struct FaceInfo { + Rect location_; + float score_; + float keypoints_[10]; + bool mask_; +} FaceInfo; + +typedef struct TrackedFaceInfo { + FaceInfo face_info_; + float iou_score_; +} TrackedFaceInfo; + +#endif + +typedef struct Point2fVector { + Point2f* points; + int length; +} Point2fVector; + +void FreePoint2fVector(Point2fVector *p); + +typedef struct RectVector { + Rect* rects; + int length; +} RectVector; + +void FreeRectVector(RectVector *p); + +typedef struct FaceInfoVector { + FaceInfo* faces; + int length; +} FaceInfoVector; + +void FreeFaceInfoVector(FaceInfoVector *p); +void NewFaceInfoVector(FaceInfoVector *v, int num); + +typedef struct TrackedFaceInfoVector { + TrackedFaceInfo* faces; + int length; +} TrackedFaceInfoVector; + +void FreeTrackedFaceInfoVector(TrackedFaceInfoVector *p); + +typedef struct FloatVector { + float* values; + int length; +} FloatVector; + +void FreeFloatVector(FloatVector *p); + +#ifdef __cplusplus +} +#endif + +#endif // !_COMMON_C_H_ diff --git a/src/common/common.hpp b/src/common/common.hpp new file mode 100644 index 0000000..5c49355 --- /dev/null +++ b/src/common/common.hpp @@ -0,0 +1,161 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include +#include +#if defined(_OPENMP) +#include +#endif + +namespace mirror { +#define kFaceFeatureDim 128 +#define kFaceNameDim 256 +const int threads_num = 2; + +// Wrapper for an individual cv::cvSize +typedef struct Size { + int width; + int height; + Size(int _width = 0, int _height = 0): width(_width), height(_height) {} +} Size; + +// Wrapper for an individual cv::cvPoint +typedef struct Point { + int x; + int y; + Point(int _x = 0, int _y = 0): x(_x), y(_y) {} + Point operator-(const Point &p2) { + return Point(x - p2.x, y - p2.y); + }; +} Point; + +// Wrapper for an individual cv::Point2f +typedef struct Point2f { + float x; + float y; + Point2f(float _x = 0, float _y = 0): x(_x), y(_y) {} +} Point2f; + +// Wrapper for an individual cv::Rect +typedef struct Rect { + int x; + int y; + int width; + int height; + Rect(int _x = 0, int _y = 0, int _width = 0, int _height = 0): x(_x), y(_y), width(_width), height(_height) {} + Point br() const { + return Point(x + width, y + height); + }; + int area() const { + return width * height; + }; + Rect operator&(const Rect &r2) const { + int inter_x = x; + int inter_y = y; + int inter_width = width; + int inter_height = height; + if (x < r2.x) { + inter_x = r2.x; + } + if (y < r2.y) { + inter_y = r2.y; + } + if (x + width > r2.x + r2.width) { + inter_width = r2.x + r2.width - inter_x; + } + if (y + height > r2.y + r2.height) { + inter_height = r2.y + r2.height - inter_y; + } + return Rect(inter_x, inter_y, inter_width, inter_height); + }; +} Rect; + +struct ImageInfo { + std::string label_; + float score_; +}; + +struct ObjectInfo { + Rect location_; + float score_; + std::string name_; +}; + +struct FaceInfo { + Rect location_; + float score_; + float keypoints_[10]; + bool mask_; +}; + +struct TrackedFaceInfo { + FaceInfo face_info_; + float iou_score_; +}; + +struct QueryResult { + std::string name_; + float sim_; +}; + +int RatioAnchors(const Rect & anchor, + const std::vector& ratios, std::vector* anchors); + +int ScaleAnchors(const std::vector& ratio_anchors, + const std::vector& scales, std::vector* anchors); + +int GenerateAnchors(const int & base_size, + const std::vector& ratios, const std::vector scales, + std::vector* anchors); + +float InterRectArea(const Rect & a, + const Rect & b); + +int ComputeIOU(const Rect & rect1, + const Rect & rect2, float * iou, + const std::string& type = "UNION"); + +template +int const NMS(const std::vector& inputs, std::vector* result, + const float& threshold, const std::string& type = "UNION") { + result->clear(); + if (inputs.size() == 0) + return -1; + + std::vector inputs_tmp; + inputs_tmp.assign(inputs.begin(), inputs.end()); + std::sort(inputs_tmp.begin(), inputs_tmp.end(), + [](const T& a, const T& b) { + return a.score_ > b.score_; + }); + + std::vector indexes(inputs_tmp.size()); + + for (int i = 0; i < indexes.size(); i++) { + indexes[i] = i; + } + + while (indexes.size() > 0) { + int good_idx = indexes[0]; + result->push_back(inputs_tmp[good_idx]); + std::vector tmp_indexes = indexes; + indexes.clear(); + for (int i = 1; i < tmp_indexes.size(); i++) { + int tmp_i = tmp_indexes[i]; + float iou = 0.0f; + ComputeIOU(inputs_tmp[good_idx].location_, inputs_tmp[tmp_i].location_, &iou, type); + if (iou <= threshold) { + indexes.push_back(tmp_i); + } + } + } + return 0; +} + +float CalculateSimilarity(const std::vector&feature1, const std::vector& feature2); +void EnlargeRect(const float& scale, Rect* rect); +void RectifyRect(Rect* rect); + +} + +#endif // !_COMMON_H_ diff --git a/src/face/detecter/anticonv/anticonv.cpp b/src/face/detecter/anticonv/anticonv.cpp new file mode 100644 index 0000000..84ddb1e --- /dev/null +++ b/src/face/detecter/anticonv/anticonv.cpp @@ -0,0 +1,136 @@ +#include "anticonv.hpp" + +#if MIRROR_VULKAN +#include "gpu.h" +#endif // MIRROR_VULKAN + +namespace mirror { +AntiConv::AntiConv() : + anticonv_net_(new ncnn::Net()), + initialized_(false) { +#if MIRROR_VULKAN + ncnn::create_gpu_instance(); + anticonv_net_->opt.use_vulkan_compute = true; +#endif // MIRROR_VULKAN + +} + +AntiConv::~AntiConv() { + if (anticonv_net_) { + anticonv_net_->clear(); + } +#if MIRROR_VULKAN + ncnn::destroy_gpu_instance(); +#endif // MIRROR_VULKAN +} + +int AntiConv::LoadModel(const char * root_path) { + std::string param_file = std::string(root_path) + "/param"; + std::string bin_file = std::string(root_path) + "/bin"; + if (anticonv_net_->load_param(param_file.c_str()) == -1 || + anticonv_net_->load_model(bin_file.c_str()) == -1) { + return 10000; + } + + // generate anchors + for (int i = 0; i < 3; ++i) { + ANCHORS anchors; + if (0 == i) { + GenerateAnchors(16, { 1.0f }, { 32, 16 }, &anchors); + } + else if (1 == i) { + GenerateAnchors(16, { 1.0f }, { 8, 4 }, &anchors); + } + else { + GenerateAnchors(16, { 1.0f }, { 2, 1 }, &anchors); + } + anchors_generated_.push_back(anchors); + } + initialized_ = true; + + return 0; +} + +int AntiConv::DetectFace(const unsigned char* rgbdata, + int img_width, int img_height, + std::vector* faces) { + faces->clear(); + if (!initialized_) { + return 10000; + } + if (rgbdata == 0) { + return 10001; + } + + float factor_x = static_cast(img_width) / inputSize_.width; + float factor_y = static_cast(img_height) / inputSize_.height; + ncnn::Extractor ex = anticonv_net_->create_extractor(); + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgbdata, + ncnn::Mat::PIXEL_RGB, img_width, img_height, inputSize_.width, inputSize_.height); + ex.input("data", in); + + std::vector faces_tmp; + for (int i = 0; i < 3; ++i) { + std::string class_layer_name = "face_rpn_cls_prob_reshape_stride" + std::to_string(RPNs_[i]); + std::string bbox_layer_name = "face_rpn_bbox_pred_stride" + std::to_string(RPNs_[i]); + std::string landmark_layer_name = "face_rpn_landmark_pred_stride" + std::to_string(RPNs_[i]); + std::string type_layer_name = "face_rpn_type_prob_reshape_stride" + std::to_string(RPNs_[i]); + + ncnn::Mat class_mat, bbox_mat, landmark_mat, type_mat; + ex.extract(class_layer_name.c_str(), class_mat); + ex.extract(bbox_layer_name.c_str(), bbox_mat); + ex.extract(landmark_layer_name.c_str(), landmark_mat); + ex.extract(type_layer_name.c_str(), type_mat); + + ANCHORS anchors = anchors_generated_.at(i); + int width = class_mat.w; + int height = class_mat.h; + int anchor_num = static_cast(anchors.size()); + for (int h = 0; h < height; ++h) { + for (int w = 0; w < width; ++w) { + int index = h * width + w; + for (int a = 0; a < anchor_num; ++a) { + float score = class_mat.channel(anchor_num + a)[index]; + if (score < scoreThreshold_) { + continue; + } + float prob = type_mat.channel(2 * anchor_num + a)[index]; + Rect box = Rect(w * RPNs_[i] + anchors[a].x, + h * RPNs_[i] + anchors[a].y, + anchors[a].width, + anchors[a].height); + + float delta_x = bbox_mat.channel(a * 4 + 0)[index]; + float delta_y = bbox_mat.channel(a * 4 + 1)[index]; + float delta_w = bbox_mat.channel(a * 4 + 2)[index]; + float delta_h = bbox_mat.channel(a * 4 + 3)[index]; + Point2f center = Point2f(box.x + box.width * 0.5f, + box.y + box.height * 0.5f); + center.x = center.x + delta_x * box.width; + center.y = center.y + delta_y * box.height; + float curr_width = exp(delta_w) * (box.width + 1); + float curr_height = exp(delta_h) * (box.height + 1); + Rect curr_box = Rect(center.x - curr_width * 0.5f, + center.y - curr_height * 0.5f, curr_width, curr_height); + curr_box.x = fmaxf(curr_box.x * factor_x, 0); + curr_box.y = fmaxf(curr_box.y * factor_y, 0); + curr_box.width = fminf(img_width - curr_box.x, curr_box.width * factor_x); + curr_box.height = fminf(img_height - curr_box.y, curr_box.height * factor_y); + + FaceInfo face_info; + memset(&face_info, 0, sizeof(face_info)); + face_info.score_ = score; + face_info.mask_ = (prob > maskThreshold_); + face_info.location_ = curr_box; + faces_tmp.push_back(face_info); + } + } + } + } + + NMS(faces_tmp, faces, iouThreshold_); + + return 0; +} + +} diff --git a/src/face/detecter/anticonv/anticonv.hpp b/src/face/detecter/anticonv/anticonv.hpp new file mode 100644 index 0000000..3815912 --- /dev/null +++ b/src/face/detecter/anticonv/anticonv.hpp @@ -0,0 +1,32 @@ +#ifndef _FACE_ANTICONV_H_ +#define _FACE_ANTICONV_H_ + +#include "../detecter.hpp" +#include "net.h" + +namespace mirror { +using ANCHORS = std::vector; +class AntiConv : public Detecter { +public: + AntiConv(); + ~AntiConv(); + int LoadModel(const char* root_path); + int DetectFace(const unsigned char* rgbdata, + int img_width, int img_height, + std::vector* faces); + +private: + ncnn::Net* anticonv_net_; + std::vector anchors_generated_; + bool initialized_; + const int RPNs_[3] = { 32, 16, 8 }; + const Size inputSize_ = { 640, 640 }; + const float iouThreshold_ = 0.4f; + const float scoreThreshold_ = 0.8f; + const float maskThreshold_ = 0.2f; + +}; + +} + +#endif // !_FACE_ANTICONV_H_ diff --git a/src/face/detecter/centerface/centerface.cpp b/src/face/detecter/centerface/centerface.cpp new file mode 100644 index 0000000..3dfb73c --- /dev/null +++ b/src/face/detecter/centerface/centerface.cpp @@ -0,0 +1,102 @@ +#include "centerface.hpp" + +#if MIRROR_VULKAN +#include "gpu.h" +#endif // MIRROR_VULKAN + +namespace mirror { +CenterFace::CenterFace() { + centernet_ = new ncnn::Net(); + initialized_ = false; +#if MIRROR_VULKAN + ncnn::create_gpu_instance(); + centernet_->opt.use_vulkan_compute = true; +#endif // MIRROR_VULKAN +} + +CenterFace::~CenterFace(){ + if (centernet_) { + centernet_->clear(); + } +#if MIRROR_VULKAN + ncnn::destroy_gpu_instance(); +#endif // MIRROR_VULKAN +} + +int CenterFace::LoadModel(const char* root_path) { + std::string param_file = std::string(root_path) + "/param"; + std::string model_file = std::string(root_path) + "/bin"; + if (centernet_->load_param(param_file.c_str()) == -1 || + centernet_->load_model(model_file.c_str()) == -1) { + return 10000; + } + + initialized_ = true; + return 0; +} + +int CenterFace::DetectFace(const unsigned char* rgbdata, + int img_width, int img_height, + std::vector* faces) { + faces->clear(); + if (!initialized_) { + return 10000; + } + if (rgbdata == 0){ + return 10001; + } + + int img_width_new = img_width / 32 * 32; + int img_height_new = img_height / 32 * 32; + float scale_x = static_cast(img_width) / img_width_new; + float scale_y = static_cast(img_height) / img_height_new; + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgbdata, ncnn::Mat::PIXEL_RGB, + img_width, img_height, img_width_new, img_height_new); + ncnn::Extractor ex = centernet_->create_extractor(); + ex.input("input.1", in); + ncnn::Mat mat_heatmap, mat_scale, mat_offset, mat_landmark; + ex.extract("537", mat_heatmap); + ex.extract("538", mat_scale); + ex.extract("539", mat_offset); + ex.extract("540", mat_landmark); + + int height = mat_heatmap.h; + int width = mat_heatmap.w; + std::vector faces_tmp; + for (int h = 0; h < height; ++h) { + for (int w = 0; w < width; ++w) { + int index = h * width + w; + float score = mat_heatmap[index]; + if (score < scoreThreshold_) { + continue; + } + float s0 = 4 * exp(mat_scale.channel(0)[index]); + float s1 = 4 * exp(mat_scale.channel(1)[index]); + float o0 = mat_offset.channel(0)[index]; + float o1 = mat_offset.channel(1)[index]; + + float ymin = fmaxf(0, 4 * (h + o0 + 0.5) - 0.5 * s0); + float xmin = fmaxf(0, 4 * (w + o1 + 0.5) - 0.5 * s1); + float ymax = fminf(ymin + s0, img_height_new); + float xmax = fminf(xmin + s1, img_width_new); + + FaceInfo face_info; + face_info.score_ = score; + face_info.location_.x = scale_x * xmin; + face_info.location_.y = scale_y * ymin; + face_info.location_.width = scale_x * (xmax - xmin); + face_info.location_.height = scale_y * (ymax - ymin); + + for (int num = 0; num < 5; ++num) { + face_info.keypoints_[num ] = scale_x * (s1 * mat_landmark.channel(2 * num + 1)[index] + xmin); + face_info.keypoints_[num + 5] = scale_y * (s0 * mat_landmark.channel(2 * num + 0)[index] + ymin); + } + faces_tmp.push_back(face_info); + } + } + NMS(faces_tmp, faces, nmsThreshold_); + return 0; +} + +} diff --git a/src/face/detecter/centerface/centerface.hpp b/src/face/detecter/centerface/centerface.hpp new file mode 100644 index 0000000..bfad3f7 --- /dev/null +++ b/src/face/detecter/centerface/centerface.hpp @@ -0,0 +1,27 @@ +#ifndef _FACE_CENTERFACE_H_ +#define _FACE_CENTERFACE_H_ + +#include "../detecter.hpp" +#include +#include "net.h" + +namespace mirror { +class CenterFace : public Detecter { +public: + CenterFace(); + ~CenterFace(); + int LoadModel(const char* root_path); + int DetectFace(const unsigned char* rgbdata, + int img_width, int img_height, + std::vector* faces); + +private: + ncnn::Net* centernet_ = nullptr; + const float scoreThreshold_ = 0.5f; + const float nmsThreshold_ = 0.5f; + bool initialized_; +}; + +} + +#endif // !_FACE_CENTERFACE_H_ diff --git a/src/face/detecter/detecter.cpp b/src/face/detecter/detecter.cpp new file mode 100644 index 0000000..66368fc --- /dev/null +++ b/src/face/detecter/detecter.cpp @@ -0,0 +1,64 @@ +#include "detecter.h" +#include "centerface/centerface.hpp" +#include "mtcnn/mtcnn.hpp" +#include "retinaface/retinaface.hpp" +#include "anticonv/anticonv.hpp" + +IDetecter new_retinaface() { + return new mirror::RetinaFace(); +} + +IDetecter new_centerface() { + return new mirror::CenterFace(); +} + +IDetecter new_mtcnn() { + return new mirror::Mtcnn(); +} + +IDetecter new_anticonv() { + return new mirror::AntiConv(); +} + +void destroy_detecter(IDetecter d) { + delete static_cast(d); +} + +int detecter_load_model(IDetecter d, const char *root_path){ + return static_cast(d)->LoadModel(root_path); +} + +int detect_face(IDetecter d, const unsigned char* rgbdata, int img_width, int img_height, FaceInfoVector* faces) { + std::vector detected; + int ret = static_cast(d)->DetectFace(rgbdata, img_width, img_height, &detected); + if (ret != 0) { + return ret; + } + + FaceInfo* fps = new FaceInfo[detected.size()]; + for (size_t i = 0; i < detected.size(); ++i) { + fps[i] = detected[i]; + } + faces->length = detected.size(); + faces->faces = fps; + return 0; +} + +namespace mirror { +Detecter* CenterfaceFactory::CreateDetecter() { + return new CenterFace(); +} + +Detecter* MtcnnFactory::CreateDetecter() { + return new Mtcnn(); +} + +Detecter* RetinafaceFactory::CreateDetecter() { + return new RetinaFace(); +} + +Detecter* AnticonvFactory::CreateDetecter() { + return new AntiConv(); +} + +} diff --git a/src/face/detecter/detecter.h b/src/face/detecter/detecter.h new file mode 100644 index 0000000..712c521 --- /dev/null +++ b/src/face/detecter/detecter.h @@ -0,0 +1,21 @@ +#ifndef _FACE_DETECTER_C_H_ +#define _FACE_DETECTER_C_H_ + +#include "../common/common.h" + +#ifdef __cplusplus +#include "detecter.hpp" +extern "C" { +#endif + typedef void* IDetecter; + IDetecter new_retinaface(); + IDetecter new_centerface(); + IDetecter new_mtcnn(); + IDetecter new_anticonv(); + void destroy_detecter(IDetecter d); + int detecter_load_model(IDetecter d, const char* root_path); + int detect_face(IDetecter d, const unsigned char* rgbdata, int img_width, int img_height, FaceInfoVector* faces); +#ifdef __cplusplus +} +#endif +#endif // !_FACE_DETECTER_C_H_ diff --git a/src/face/detecter/detecter.hpp b/src/face/detecter/detecter.hpp new file mode 100644 index 0000000..473322d --- /dev/null +++ b/src/face/detecter/detecter.hpp @@ -0,0 +1,58 @@ +#ifndef _FACE_DETECTER_H_ +#define _FACE_DETECTER_H_ + +#include "../common/common.hpp" + +namespace mirror { +// 抽象类 +class Detecter { +public: + virtual ~Detecter() {}; + virtual int LoadModel(const char* root_path) = 0; + virtual int DetectFace(const unsigned char* rgbdata, + int img_width, int img_height, + std::vector* faces) = 0; + +}; + +// 工厂基类 +class DetecterFactory { +public: + virtual Detecter* CreateDetecter() = 0; + virtual ~DetecterFactory() {}; +}; + +// 不同人脸检测器 +class CenterfaceFactory : public DetecterFactory { +public: + CenterfaceFactory() {} + ~CenterfaceFactory() {} + Detecter* CreateDetecter(); +}; + +class MtcnnFactory : public DetecterFactory { +public: + MtcnnFactory() {} + ~MtcnnFactory() {} + Detecter* CreateDetecter(); + +}; + +class RetinafaceFactory : public DetecterFactory { +public: + RetinafaceFactory() {} + ~RetinafaceFactory() {} + Detecter* CreateDetecter(); +}; + +class AnticonvFactory : public DetecterFactory { +public: + AnticonvFactory() {} + ~AnticonvFactory() {} + Detecter* CreateDetecter(); +}; + +} + +#endif // !_FACE_DETECTER_H_ + diff --git a/src/face/detecter/mtcnn/mtcnn.cpp b/src/face/detecter/mtcnn/mtcnn.cpp new file mode 100644 index 0000000..924454f --- /dev/null +++ b/src/face/detecter/mtcnn/mtcnn.cpp @@ -0,0 +1,260 @@ +#include "mtcnn.hpp" + +#if MIRROR_VULKAN +#include "gpu.h" +#endif // MIRROR_VULKAN + +namespace mirror { +Mtcnn::Mtcnn() : + pnet_(new ncnn::Net()), + rnet_(new ncnn::Net()), + onet_(new ncnn::Net()), + pnet_size_(12), + min_face_size_(40), + scale_factor_(0.709f), + initialized_(false) { +#if MIRROR_VULKAN + ncnn::create_gpu_instance(); + pnet_->opt.use_vulkan_compute = true; + rnet_->opt.use_vulkan_compute = true; + onet_->opt.use_vulkan_compute = true; +#endif // MIRROR_VULKAN +} + +Mtcnn::~Mtcnn() { + if (pnet_) { + pnet_->clear(); + } + if (rnet_) { + rnet_->clear(); + } + if (onet_) { + onet_->clear(); + } +#if MIRROR_VULKAN + ncnn::destroy_gpu_instance(); +#endif // MIRROR_VULKAN +} + +int Mtcnn::LoadModel(const char * root_path) { + std::string pnet_param = std::string(root_path) + "/pnet.param"; + std::string pnet_bin = std::string(root_path) + "/pnet.bin"; + if (pnet_->load_param(pnet_param.c_str()) == -1 || + pnet_->load_model(pnet_bin.c_str()) == -1) { + return 10000; + } + std::string rnet_param = std::string(root_path) + "/rnet.param"; + std::string rnet_bin = std::string(root_path) + "/rnet.bin"; + if (rnet_->load_param(rnet_param.c_str()) == -1 || + rnet_->load_model(rnet_bin.c_str()) == -1) { + return 10000; + } + std::string onet_param = std::string(root_path) + "/onet.param"; + std::string onet_bin = std::string(root_path) + "/onet.bin"; + if (onet_->load_param(onet_param.c_str()) == -1 || + onet_->load_model(onet_bin.c_str()) == -1) { + return 10000; + } + initialized_ = true; + return 0; +} + +int Mtcnn::DetectFace(const unsigned char* rgbdata, + int img_width, int img_height, + std::vector* faces) { + if (rgbdata == 0) { + return 10001; + } + if (!initialized_) { + return 10000; + } + Size max_size = Size(img_width, img_height); + ncnn::Mat img_in = ncnn::Mat::from_pixels(rgbdata, + ncnn::Mat::PIXEL_RGB, img_width, img_height); + img_in.substract_mean_normalize(meanVals, normVals); + + std::vector first_bboxes, second_bboxes; + std::vector first_bboxes_result; + PDetect(img_in, &first_bboxes); + NMS(first_bboxes, &first_bboxes_result, nms_threshold_[0]); + Refine(&first_bboxes_result, max_size); + + RDetect(img_in, first_bboxes_result, &second_bboxes); + std::vector second_bboxes_result; + NMS(second_bboxes, &second_bboxes_result, nms_threshold_[1]); + Refine(&second_bboxes_result, max_size); + + std::vector third_bboxes; + ODetect(img_in, second_bboxes_result, &third_bboxes); + NMS(third_bboxes, faces, nms_threshold_[2], "MIN"); + Refine(faces, max_size); + return 0; +} + +int Mtcnn::PDetect(const ncnn::Mat & img_in, + std::vector* first_bboxes) { + first_bboxes->clear(); + int width = img_in.w; + int height = img_in.h; + float min_side = fminf(width, height); + float curr_scale = float(pnet_size_) / min_face_size_; + min_side *= curr_scale; + std::vector scales; + while (min_side > pnet_size_) { + scales.push_back(curr_scale); + min_side *= scale_factor_; + curr_scale *= scale_factor_; + } + + // mutiscale resize the image + for (int i = 0; i < static_cast(scales.size()); ++i) { + int w = static_cast(width * scales[i]); + int h = static_cast(height * scales[i]); + ncnn::Mat img_resized; + ncnn::resize_bilinear(img_in, img_resized, w, h); + ncnn::Extractor ex = pnet_->create_extractor(); + //ex.set_num_threads(2); + ex.set_light_mode(true); + ex.input("data", img_resized); + ncnn::Mat score_mat, location_mat; + ex.extract("prob1", score_mat); + ex.extract("conv4-2", location_mat); + const int stride = 2; + const int cell_size = 12; + for (int h = 0; h < score_mat.h; ++h) { + for (int w = 0; w < score_mat.w; ++w) { + int index = h * score_mat.w + w; + // pnet output: 1x1x2 no-face && face + // face score: channel(1) + float score = score_mat.channel(1)[index]; + if (score < threshold_[0]) continue; + + // 1. generated bounding box + int x1 = round((stride * w + 1) / scales[i]); + int y1 = round((stride * h + 1) / scales[i]); + int x2 = round((stride * w + 1 + cell_size) / scales[i]); + int y2 = round((stride * h + 1 + cell_size) / scales[i]); + + // 2. regression bounding box + float x1_reg = location_mat.channel(0)[index]; + float y1_reg = location_mat.channel(1)[index]; + float x2_reg = location_mat.channel(2)[index]; + float y2_reg = location_mat.channel(3)[index]; + + int bbox_width = x2 - x1 + 1; + int bbox_height = y2 - y1 + 1; + + FaceInfo face_info; + face_info.score_ = score; + face_info.location_.x = x1 + x1_reg * bbox_width; + face_info.location_.y = y1 + y1_reg * bbox_height; + face_info.location_.width = x2 + x2_reg * bbox_width - face_info.location_.x; + face_info.location_.height = y2 + y2_reg * bbox_height - face_info.location_.y; + face_info.location_ = face_info.location_ & Rect(0, 0, width, height); + first_bboxes->push_back(face_info); + } + } + } + return 0; +} + +int Mtcnn::RDetect(const ncnn::Mat & img_in, + const std::vector& first_bboxes, + std::vector* second_bboxes) { + second_bboxes->clear(); + for (int i = 0; i < static_cast(first_bboxes.size()); ++i) { + Rect face = first_bboxes.at(i).location_ & Rect(0, 0, img_in.w, img_in.h); + ncnn::Mat img_face, img_resized; + ncnn::copy_cut_border(img_in, img_face, face.y, img_in.h - face.br().y, face.x, img_in.w - face.br().x); + ncnn::resize_bilinear(img_face, img_resized, 24, 24); + ncnn::Extractor ex = rnet_->create_extractor(); + ex.set_light_mode(true); + ex.set_num_threads(2); + ex.input("data", img_resized); + ncnn::Mat score_mat, location_mat; + ex.extract("prob1", score_mat); + ex.extract("conv5-2", location_mat); + float score = score_mat[1]; + if (score < threshold_[1]) continue; + float x_reg = location_mat[0]; + float y_reg = location_mat[1]; + float w_reg = location_mat[2]; + float h_reg = location_mat[3]; + + FaceInfo face_info; + face_info.score_ = score; + face_info.location_.x = face.x + x_reg * face.width; + face_info.location_.y = face.y + y_reg * face.height; + face_info.location_.width = face.x + face.width + + w_reg * face.width - face_info.location_.x; + face_info.location_.height = face.y + face.height + + h_reg * face.height - face_info.location_.y; + second_bboxes->push_back(face_info); + } + return 0; +} + +int Mtcnn::ODetect(const ncnn::Mat & img_in, + const std::vector& second_bboxes, + std::vector* third_bboxes) { + third_bboxes->clear(); + for (int i = 0; i < static_cast(second_bboxes.size()); ++i) { + Rect face = second_bboxes.at(i).location_ & Rect(0, 0, img_in.w, img_in.h); + ncnn::Mat img_face, img_resized; + ncnn::copy_cut_border(img_in, img_face, face.y, img_in.h - face.br().y, face.x, img_in.w - face.br().x); + ncnn::resize_bilinear(img_face, img_resized, 48, 48); + + ncnn::Extractor ex = onet_->create_extractor(); + ex.set_light_mode(true); + ex.set_num_threads(2); + ex.input("data", img_resized); + ncnn::Mat score_mat, location_mat, keypoints_mat; + ex.extract("prob1", score_mat); + ex.extract("conv6-2", location_mat); + ex.extract("conv6-3", keypoints_mat); + float score = score_mat[1]; + if (score < threshold_[1]) continue; + float x_reg = location_mat[0]; + float y_reg = location_mat[1]; + float w_reg = location_mat[2]; + float h_reg = location_mat[3]; + + FaceInfo face_info; + face_info.score_ = score; + face_info.location_.x = face.x + x_reg * face.width; + face_info.location_.y = face.y + y_reg * face.height; + face_info.location_.width = face.x + face.width + + w_reg * face.width - face_info.location_.x; + face_info.location_.height = face.y + face.height + + h_reg * face.height - face_info.location_.y; + + for (int num = 0; num < 5; num++) { + face_info.keypoints_[num] = face.x + face.width * keypoints_mat[num]; + face_info.keypoints_[num + 5] = face.y + face.height * keypoints_mat[num + 5]; + } + + third_bboxes->push_back(face_info); + } + return 0; +} + +int Mtcnn::Refine(std::vector* bboxes, const Size max_size) { + int num_boxes = static_cast(bboxes->size()); + for (int i = 0; i < num_boxes; ++i) { + FaceInfo face_info = bboxes->at(i); + int width = face_info.location_.width; + int height = face_info.location_.height; + float max_side = fmaxf(width, height); + + face_info.location_.x = face_info.location_.x + 0.5 * width - 0.5 * max_side; + face_info.location_.y = face_info.location_.y + 0.5 * height - 0.5 * max_side; + face_info.location_.width = max_side; + face_info.location_.height = max_side; + face_info.location_ = face_info.location_ & Rect(0, 0, max_size.width, max_size.height); + bboxes->at(i) = face_info; + } + + return 0; +} + +} diff --git a/src/face/detecter/mtcnn/mtcnn.hpp b/src/face/detecter/mtcnn/mtcnn.hpp new file mode 100644 index 0000000..0a1bb6d --- /dev/null +++ b/src/face/detecter/mtcnn/mtcnn.hpp @@ -0,0 +1,46 @@ +#ifndef _FACE_MTCNN_H_ +#define _FACE_MTCNN_H_ + +#include "../detecter.hpp" +#include +#include "net.h" + +namespace mirror { +class Mtcnn : public Detecter { +public: + Mtcnn(); + ~Mtcnn(); + int LoadModel(const char* root_path); + int DetectFace(const unsigned char* rgbdata, + int img_width, int img_height, + std::vector* faces); + +private: + ncnn::Net* pnet_ = nullptr; + ncnn::Net* rnet_ = nullptr; + ncnn::Net* onet_ = nullptr; + int pnet_size_; + int min_face_size_; + float scale_factor_; + bool initialized_; + const float meanVals[3] = { 127.5f, 127.5f, 127.5f }; + const float normVals[3] = { 0.0078125f, 0.0078125f, 0.0078125f }; + const float nms_threshold_[3] = { 0.5f, 0.7f, 0.7f }; + const float threshold_[3] = { 0.8f, 0.8f, 0.6f }; + +private: + int PDetect(const ncnn::Mat& img_in, std::vector* first_bboxes); + int RDetect(const ncnn::Mat& img_in, const std::vector& first_bboxes, + std::vector* second_bboxes); + int ODetect(const ncnn::Mat& img_in, + const std::vector& second_bboxes, + std::vector* third_bboxes); + + int Refine(std::vector* bboxes, const Size max_size); +}; + +} + + +#endif // !_FACE_MTCNN_H_ + diff --git a/src/face/detecter/retinaface/retinaface.cpp b/src/face/detecter/retinaface/retinaface.cpp new file mode 100644 index 0000000..9bf1627 --- /dev/null +++ b/src/face/detecter/retinaface/retinaface.cpp @@ -0,0 +1,151 @@ +#include "retinaface.hpp" +#include + +#if MIRROR_VULKAN +#include "gpu.h" +#endif // MIRROR_VULKAN + +namespace mirror { +RetinaFace::RetinaFace() : + retina_net_(new ncnn::Net()), + initialized_(false) { +#if MIRROR_VULKAN + ncnn::create_gpu_instance(); + retina_net_->opt.use_vulkan_compute = true; +#endif // MIRROR_VULKAN + +} + +RetinaFace::~RetinaFace() { + if (retina_net_) { + retina_net_->clear(); + } +#if MIRROR_VULKAN + ncnn::destroy_gpu_instance(); +#endif // MIRROR_VULKAN +} + +int RetinaFace::LoadModel(const char * root_path) { + std::string fd_param = std::string(root_path) + "/param"; + std::string fd_bin = std::string(root_path) + "/bin"; + if (retina_net_->load_param(fd_param.c_str()) == -1 || + retina_net_->load_model(fd_bin.c_str()) == -1) { + return 10000; + } + + // generate anchors + for (int i = 0; i < 3; ++i) { + ANCHORS anchors; + if (0 == i) { + GenerateAnchors(16, { 1.0f }, { 32, 16 }, &anchors); + } + else if (1 == i) { + GenerateAnchors(16, { 1.0f }, { 8, 4 }, &anchors); + } + else { + GenerateAnchors(16, { 1.0f }, { 2, 1 }, &anchors); + } + anchors_generated_.push_back(anchors); + } + initialized_ = true; + + return 0; +} + +int RetinaFace::DetectFace(const unsigned char* rgbdata, + int img_width, int img_height, + std::vector* faces) { + faces->clear(); + if (!initialized_) { + return 10000; + } + if (rgbdata == 0) { + return 10001; + } + + float factor_x = static_cast(img_width) / inputSize_.width; + float factor_y = static_cast(img_height) / inputSize_.height; + ncnn::Extractor ex = retina_net_->create_extractor(); + ncnn::Mat in = ncnn::Mat::from_pixels_resize(rgbdata, + ncnn::Mat::PIXEL_RGB, img_width, img_height, inputSize_.width, inputSize_.height); + ex.input("data", in); + + std::vector faces_tmp; + for (int i = 0; i < 3; ++i) { + std::string class_layer_name = "face_rpn_cls_prob_reshape_stride" + std::to_string(RPNs_[i]); + std::string bbox_layer_name = "face_rpn_bbox_pred_stride" + std::to_string(RPNs_[i]); + std::string landmark_layer_name = "face_rpn_landmark_pred_stride" + std::to_string(RPNs_[i]); + + ncnn::Mat class_mat, bbox_mat, landmark_mat; + ex.extract(class_layer_name.c_str(), class_mat); + ex.extract(bbox_layer_name.c_str(), bbox_mat); + ex.extract(landmark_layer_name.c_str(), landmark_mat); + + ANCHORS anchors = anchors_generated_.at(i); + int width = class_mat.w; + int height = class_mat.h; + int anchor_num = static_cast(anchors.size()); + for (int h = 0; h < height; ++h) { + for (int w = 0; w < width; ++w) { + int index = h * width + w; + for (int a = 0; a < anchor_num; ++a) { + float score = class_mat.channel(anchor_num + a)[index]; + if (score < scoreThreshold_) { + continue; + } + // 1.获取anchor生成的box + Rect box = Rect(w * RPNs_[i] + anchors[a].x, + h * RPNs_[i] + anchors[a].y, + anchors[a].width, + anchors[a].height); + + // 2.解析出偏移量 + float delta_x = bbox_mat.channel(a * 4 + 0)[index]; + float delta_y = bbox_mat.channel(a * 4 + 1)[index]; + float delta_w = bbox_mat.channel(a * 4 + 2)[index]; + float delta_h = bbox_mat.channel(a * 4 + 3)[index]; + + // 3.计算anchor box的中心 + Point2f center = Point2f(box.x + box.width * 0.5f, + box.y + box.height * 0.5f); + + // 4.计算框的实际中心（anchor的中心+偏移量） + center.x = center.x + delta_x * box.width; + center.y = center.y + delta_y * box.height; + + // 5.计算出实际的宽和高 + float curr_width = exp(delta_w) * (box.width + 1); + float curr_height = exp(delta_h) * (box.height + 1); + + // 6.获取实际的矩形位置 + Rect curr_box = Rect(center.x - curr_width * 0.5f, + center.y - curr_height * 0.5f, curr_width, curr_height); + curr_box.x = fmaxf(curr_box.x * factor_x, 0); + curr_box.y = fmaxf(curr_box.y * factor_y, 0); + curr_box.width = fminf(img_width - curr_box.x, curr_box.width * factor_x); + curr_box.height = fminf(img_height - curr_box.y, curr_box.height * factor_y); + + FaceInfo face_info; + memset(&face_info, 0, sizeof(face_info)); + + int offset_index = landmark_mat.c / anchor_num; + for (int k = 0; k < 5; ++k) { + float x = landmark_mat.channel(a * offset_index + 2 * k)[index] * box.width + center.x; + float y = landmark_mat.channel(a * offset_index + 2 * k + 1)[index] * box.height + center.y; + face_info.keypoints_[k] = fminf(fmaxf(x * factor_x, 0.0f), img_width - 1); + face_info.keypoints_[k + 5] = fminf(fmaxf(y * factor_y, 0.0f), img_height - 1); + } + + face_info.score_ = score; + face_info.location_ = curr_box; + faces_tmp.push_back(face_info); + } + } + } + } + + NMS(faces_tmp, faces, iouThreshold_); + return 0; +} + +} diff --git a/src/face/detecter/retinaface/retinaface.hpp b/src/face/detecter/retinaface/retinaface.hpp new file mode 100644 index 0000000..3e164e7 --- /dev/null +++ b/src/face/detecter/retinaface/retinaface.hpp @@ -0,0 +1,32 @@ +#ifndef _RETINAFACE_H_ +#define _RETINAFACE_H_ + +#include "../detecter.hpp" +#include "net.h" + +namespace mirror { +using ANCHORS = std::vector; +class RetinaFace : public Detecter { +public: + RetinaFace(); + ~RetinaFace(); + int LoadModel(const char* root_path); + int DetectFace(const unsigned char* rgbdata, + int img_width, int img_height, + std::vector* faces); + +private: + ncnn::Net* retina_net_; + std::vector anchors_generated_; + bool initialized_; + const int RPNs_[3] = { 32, 16, 8 }; + const Size inputSize_ = { 300, 300 }; + const float iouThreshold_ = 0.4f; + const float scoreThreshold_ = 0.8f; + +}; + +} + +#endif // !_RETINAFACE_H_ + diff --git a/src/face/landmarker/insightface/insightface.cpp b/src/face/landmarker/insightface/insightface.cpp new file mode 100644 index 0000000..3fc0db3 --- /dev/null +++ b/src/face/landmarker/insightface/insightface.cpp @@ -0,0 +1,87 @@ +#include "insightface.hpp" +#include +#include "../../common/common.hpp" + +#if MIRROR_VULKAN +#include "gpu.h" +#endif // MIRROR_VULKAN + +namespace mirror { +InsightfaceLandmarker::InsightfaceLandmarker() { + insightface_landmarker_net_ = new ncnn::Net(); + initialized = false; +#if MIRROR_VULKAN + ncnn::create_gpu_instance(); + insightface_landmarker_net_->opt.use_vulkan_compute = true; +#endif // MIRROR_VULKAN +} + +InsightfaceLandmarker::~InsightfaceLandmarker() { + insightface_landmarker_net_->clear(); +#if MIRROR_VULKAN + ncnn::destroy_gpu_instance(); +#endif // MIRROR_VULKAN +} + +int InsightfaceLandmarker::LoadModel(const char * root_path) { + std::string fl_param = std::string(root_path) + "/param"; + std::string fl_bin = std::string(root_path) + "/bin"; + if (insightface_landmarker_net_->load_param(fl_param.c_str()) == -1 || + insightface_landmarker_net_->load_model(fl_bin.c_str()) == -1) { + return 10000; + } + initialized = true; + return 0; +} + +int InsightfaceLandmarker::ExtractKeypoints(const unsigned char* rgbdata, + int img_width, int img_height, + const Rect & face, std::vector* keypoints) { + keypoints->clear(); + if (!initialized) { + return 10000; + } + + if (rgbdata == 0){ + return 10001; + } + + // 1 enlarge the face rect + Rect face_enlarged = face; + const float enlarge_scale = 1.5f; + EnlargeRect(enlarge_scale, &face_enlarged); + + // 2 square the rect + RectifyRect(&face_enlarged); + face_enlarged = face_enlarged & Rect(0, 0, img_width, img_height); + + // 3 crop the face + size_t total_size = face_enlarged.width * face_enlarged.height * 3 * sizeof(unsigned char); + unsigned char* img_face = (unsigned char*)malloc(total_size); + const unsigned char *start_ptr = rgbdata; + for(size_t i = 0; i < face_enlarged.height; ++i) { + const unsigned char* srcCursor = start_ptr + ((i + face_enlarged.y) * img_width + face_enlarged.x) * 3; + unsigned char* dstCursor = img_face + i * face_enlarged.width * 3; + memcpy(dstCursor, srcCursor, sizeof(unsigned char) * 3 * face_enlarged.width); + } + + // 4 do inference + ncnn::Extractor ex = insightface_landmarker_net_->create_extractor(); + ncnn::Mat in = ncnn::Mat::from_pixels_resize(img_face, + ncnn::Mat::PIXEL_RGB, face_enlarged.width, face_enlarged.height, 192, 192); + ex.input("data", in); + ncnn::Mat out; + ex.extract("fc1", out); + + for (int i = 0; i < 106; ++i) { + float x = (out[2 * i] + 1.0f) * face_enlarged.width / 2 + face_enlarged.x; + float y = (out[2 * i + 1] + 1.0f) * face_enlarged.height / 2 + face_enlarged.y; + keypoints->push_back(Point2f(x, y)); + } + + free(img_face); + + return 0; +} + +} diff --git a/src/face/landmarker/insightface/insightface.hpp b/src/face/landmarker/insightface/insightface.hpp new file mode 100644 index 0000000..270f3ff --- /dev/null +++ b/src/face/landmarker/insightface/insightface.hpp @@ -0,0 +1,26 @@ +#ifndef _FACE_INSIGHTFACE_LANDMARKER_H_ +#define _FACE_INSIGHTFACE_LANDMARKER_H_ + +#include "../landmarker.hpp" +#include "net.h" + +namespace mirror { +class InsightfaceLandmarker : public Landmarker { +public: + InsightfaceLandmarker(); + ~InsightfaceLandmarker(); + + int LoadModel(const char* root_path); + int ExtractKeypoints(const unsigned char* rgbdata, + int img_width, int img_height, + const Rect& face, std::vector* keypoints); + +private: + ncnn::Net* insightface_landmarker_net_; + bool initialized; +}; + +} + +#endif // !_FACE_INSIGHTFACE_LANDMARKER_H_ + diff --git a/src/face/landmarker/landmarker.cpp b/src/face/landmarker/landmarker.cpp new file mode 100644 index 0000000..fe1af11 --- /dev/null +++ b/src/face/landmarker/landmarker.cpp @@ -0,0 +1,50 @@ +#include "landmarker.h" +#include "zqlandmarker/zqlandmarker.hpp" +#include "insightface/insightface.hpp" + +ILandmarker new_zq() { + return new mirror::ZQLandmarker(); +} + +ILandmarker new_insightface() { + return new mirror::InsightfaceLandmarker(); +} + +void destroy_landmarker(ILandmarker m) { + delete static_cast(m); +} + +int landmarker_load_model(ILandmarker m, const char *root_path) { + return static_cast(m)->LoadModel(root_path); +} + +int extract_keypoints( + ILandmarker m, + const unsigned char* rgbdata, + int img_width, + int img_height, + const Rect* face, + Point2fVector* keypoints) { + std::vector points; + int ret = static_cast(m)->ExtractKeypoints(rgbdata, img_width, img_height, *face, &points); + if (ret != 0) { + return ret; + } + keypoints->length = points.size(); + keypoints->points = (Point2f *)malloc(keypoints->length * sizeof(Point2f)); + for (size_t i = 0; i < points.size(); ++i) { + keypoints->points[i] = points[i]; + } + return 0; +} + +namespace mirror { +Landmarker* ZQLandmarkerFactory::CreateLandmarker() { + return new ZQLandmarker(); +} + +Landmarker* InsightfaceLandmarkerFactory::CreateLandmarker() { + return new InsightfaceLandmarker(); +} + +} diff --git a/src/face/landmarker/landmarker.h b/src/face/landmarker/landmarker.h new file mode 100644 index 0000000..780c03c --- /dev/null +++ b/src/face/landmarker/landmarker.h @@ -0,0 +1,19 @@ +#ifndef _FACE_LANDMARKER_C_H_ +#define _FACE_LANDMARKER_C_H_ + +#include "../common/common.h" + +#ifdef __cplusplus +#include "landmarker.hpp" +extern "C" { +#endif + typedef void* ILandmarker; + ILandmarker new_insightface(); + ILandmarker new_zq(); + void destroy_landmarker(ILandmarker m); + int landmarker_load_model(ILandmarker m, const char* root_path); + int extract_keypoints(ILandmarker m, const unsigned char* rgbdata, int img_width, int img_height, const Rect* face, Point2fVector* keypoints); +#ifdef __cplusplus +} +#endif +#endif // !_FACE_LANDMARKER_C_H_ diff --git a/src/face/landmarker/landmarker.hpp b/src/face/landmarker/landmarker.hpp new file mode 100644 index 0000000..ce9e7a8 --- /dev/null +++ b/src/face/landmarker/landmarker.hpp @@ -0,0 +1,42 @@ +#ifndef _FACE_LANDMARKER_H_ +#define _FACE_LANDMARKER_H_ + +#include "../common/common.hpp" + +namespace mirror { +// 抽象类 +class Landmarker { +public: + virtual ~Landmarker() {}; + virtual int LoadModel(const char* root_path) = 0; + virtual int ExtractKeypoints(const unsigned char* rgbdata, + int img_width, int img_height, + const Rect& face, std::vector* keypoints) = 0; +}; + +// 工厂基类 +class LandmarkerFactory { +public: + virtual Landmarker* CreateLandmarker() = 0; + virtual ~LandmarkerFactory() {} +}; + +// 不同landmark检测器工厂 +class ZQLandmarkerFactory : public LandmarkerFactory { +public: + ZQLandmarkerFactory(){} + Landmarker* CreateLandmarker(); + ~ZQLandmarkerFactory() {} +}; + +class InsightfaceLandmarkerFactory : public LandmarkerFactory { +public: + InsightfaceLandmarkerFactory(){} + Landmarker* CreateLandmarker(); + ~InsightfaceLandmarkerFactory() {} +}; + +} + +#endif // !_FACE_LANDMARKER_H_ + diff --git a/src/face/landmarker/zqlandmarker/zqlandmarker.cpp b/src/face/landmarker/zqlandmarker/zqlandmarker.cpp new file mode 100644 index 0000000..2b8b0ce --- /dev/null +++ b/src/face/landmarker/zqlandmarker/zqlandmarker.cpp @@ -0,0 +1,75 @@ +#include "zqlandmarker.hpp" +#include + +#if MIRROR_VULKAN +#include "gpu.h" +#endif // MIRROR_VULKAN + +namespace mirror { +ZQLandmarker::ZQLandmarker() { + zq_landmarker_net_ = new ncnn::Net(); + initialized = false; +#if MIRROR_VULKAN + ncnn::create_gpu_instance(); + zq_landmarker_net_->opt.use_vulkan_compute = true; +#endif // MIRROR_VULKAN +} + +ZQLandmarker::~ZQLandmarker() { + zq_landmarker_net_->clear(); +#if MIRROR_VULKAN + ncnn::destroy_gpu_instance(); +#endif // MIRROR_VULKAN +} + +int ZQLandmarker::LoadModel(const char * root_path) { + std::string fl_param = std::string(root_path) + "/param"; + std::string fl_bin = std::string(root_path) + "/bin"; + if (zq_landmarker_net_->load_param(fl_param.c_str()) == -1 || + zq_landmarker_net_->load_model(fl_bin.c_str()) == -1) { + return 10000; + } + initialized = true; + return 0; +} + +int ZQLandmarker::ExtractKeypoints(const unsigned char* rgbdata, + int img_width, int img_height, + const Rect & face, std::vector* keypoints) { + keypoints->clear(); + if (!initialized) { + return 10000; + } + + if (rgbdata == 0){ + return 10001; + } + + size_t total_size = face.width * face.height * 3 * sizeof(unsigned char); + unsigned char* img_face = (unsigned char*)malloc(total_size); + const unsigned char *start_ptr = rgbdata; + for(size_t i = 0; i < face.height; ++i) { + const unsigned char* srcCursor = start_ptr + ((i + face.y) * img_width + face.x) * 3; + unsigned char* dstCursor = img_face + i * face.width * 3; + memcpy(dstCursor, srcCursor, sizeof(unsigned char) * 3 * face.width); + } + ncnn::Extractor ex = zq_landmarker_net_->create_extractor(); + ncnn::Mat in = ncnn::Mat::from_pixels_resize(img_face, + ncnn::Mat::PIXEL_RGB, face.width, face.height, 112, 112); + in.substract_mean_normalize(meanVals, normVals); + ex.input("data", in); + ncnn::Mat out; + ex.extract("bn6_3", out); + + for (int i = 0; i < 106; ++i) { + float x = abs(out[2 * i] * face.width) + face.x; + float y = abs(out[2 * i + 1] * face.height) + face.y; + keypoints->push_back(Point2f(x, y)); + } + + + free(img_face); + return 0; +} + +} diff --git a/src/face/landmarker/zqlandmarker/zqlandmarker.hpp b/src/face/landmarker/zqlandmarker/zqlandmarker.hpp new file mode 100644 index 0000000..cf3b849 --- /dev/null +++ b/src/face/landmarker/zqlandmarker/zqlandmarker.hpp @@ -0,0 +1,28 @@ +#ifndef _FACE_ZQLANDMARKER_H_ +#define _FACE_ZQLANDMARKER_H_ + +#include "../landmarker.hpp" +#include "net.h" + +namespace mirror { +class ZQLandmarker : public Landmarker { +public: + ZQLandmarker(); + ~ZQLandmarker(); + + int LoadModel(const char* root_path); + int ExtractKeypoints(const unsigned char* rgbdata, + int img_width, int img_height, + const Rect& face, std::vector* keypoints); + +private: + ncnn::Net* zq_landmarker_net_; + const float meanVals[3] = { 127.5f, 127.5f, 127.5f }; + const float normVals[3] = { 0.0078125f, 0.0078125f, 0.0078125f }; + bool initialized; +}; + +} + +#endif // !_FACE_ZQLANDMARKER_H_ + diff --git a/src/face/recognizer/mobilefacenet/mobilefacenet.cpp b/src/face/recognizer/mobilefacenet/mobilefacenet.cpp new file mode 100644 index 0000000..47553a2 --- /dev/null +++ b/src/face/recognizer/mobilefacenet/mobilefacenet.cpp @@ -0,0 +1,73 @@ +#include "mobilefacenet.hpp" +#include +#if MIRROR_VULKAN +#include "gpu.h" +#endif // MIRROR_VULKAN + +namespace mirror { +Mobilefacenet::Mobilefacenet() { + mobileface_net_ = new ncnn::Net(); + initialized_ = false; +#if MIRROR_VULKAN + ncnn::create_gpu_instance(); + mobileface_net_->opt.use_vulkan_compute = true; +#endif // MIRROR_VULKAN +} + +Mobilefacenet::~Mobilefacenet() { + mobileface_net_->clear(); +#if MIRROR_VULKAN + ncnn::destroy_gpu_instance(); +#endif // MIRROR_VULKAN +} + +int Mobilefacenet::LoadModel(const char * root_path) { + std::string param_file = std::string(root_path) + "/param"; + std::string bin_file = std::string(root_path) + "/bin"; + if (mobileface_net_->load_param(param_file.c_str()) == -1 || + mobileface_net_->load_model(bin_file.c_str()) == -1) { + return 10000; + } + + initialized_ = true; + return 0; +} + +int Mobilefacenet::ExtractFeature(const unsigned char* rgbdata, + int img_width, int img_height, + const Rect & face, std::vector* feature) { + feature->clear(); + if (!initialized_) { + return 10000; + } + if (rgbdata == 0){ + return 10001; + } + + size_t total_size = face.width * face.height * 3 * sizeof(unsigned char); + unsigned char* img_face = (unsigned char*)malloc(total_size); + const unsigned char *start_ptr = rgbdata; + for(size_t i = 0; i < face.height; ++i) { + const unsigned char* srcCursor = start_ptr + ((i + face.y) * img_width + face.x) * 3; + unsigned char* dstCursor = img_face + i * face.width * 3; + memcpy(dstCursor, srcCursor, sizeof(unsigned char) * 3 * face.width); + } + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(img_face, + ncnn::Mat::PIXEL_RGB, face.width, face.height, 112, 112); + feature->resize(kFaceFeatureDim); + ncnn::Extractor ex = mobileface_net_->create_extractor(); + ex.input("data", in); + ncnn::Mat out; + ex.extract("fc1", out); + for (int i = 0; i < kFaceFeatureDim; ++i) { + feature->at(i) = out[i]; + } + + free(img_face); + return 0; +} + +} + + diff --git a/src/face/recognizer/mobilefacenet/mobilefacenet.hpp b/src/face/recognizer/mobilefacenet/mobilefacenet.hpp new file mode 100644 index 0000000..c790d1c --- /dev/null +++ b/src/face/recognizer/mobilefacenet/mobilefacenet.hpp @@ -0,0 +1,29 @@ +#ifndef _FACE_MOBILEFACENET_H_ +#define _FACE_MOBILEFACENET_H_ + +#include "../recognizer.hpp" +#include +#include "net.h" + +namespace mirror { + +class Mobilefacenet : public Recognizer { +public: + Mobilefacenet(); + ~Mobilefacenet(); + + int LoadModel(const char* root_path); + int ExtractFeature(const unsigned char* rgbdata, + int img_width, int img_height, + const Rect& face, + std::vector* feature); + +private: + ncnn::Net* mobileface_net_; + bool initialized_; +}; + +} + +#endif // !_FACE_MOBILEFACENET_H_ + diff --git a/src/face/recognizer/recognizer.cpp b/src/face/recognizer/recognizer.cpp new file mode 100644 index 0000000..aec578c --- /dev/null +++ b/src/face/recognizer/recognizer.cpp @@ -0,0 +1,35 @@ +#include "recognizer.h" +#include "./mobilefacenet/mobilefacenet.hpp" + +IRecognizer new_mobilefacenet() { + return new mirror::Mobilefacenet(); +} + +void destroy_recognizer(IRecognizer r) { + delete static_cast(r); +} + +int recognizer_load_model(IRecognizer r, const char* root_path) { + return static_cast(r)->LoadModel(root_path); +} + +int extract_feature(IRecognizer r, const unsigned char* rgbdata, int img_width, int img_height, const Rect* face, FloatVector* feature) { + std::vectorfeatures; + int ret = static_cast(r)->ExtractFeature(rgbdata, img_width, img_height, *face, &features); + if (ret != 0) { + return ret; + } + feature->length = features.size(); + feature->values = (float*)malloc(feature->length * sizeof(float)); + for (size_t i = 0; i < feature->length; ++i) { + feature->values[i] = features.at(i); + } + return 0; +} + +namespace mirror { +Recognizer* MobilefacenetRecognizerFactory::CreateRecognizer() { + return new Mobilefacenet(); +} + +} diff --git a/src/face/recognizer/recognizer.h b/src/face/recognizer/recognizer.h new file mode 100644 index 0000000..04e7a58 --- /dev/null +++ b/src/face/recognizer/recognizer.h @@ -0,0 +1,17 @@ +#ifndef _FACE_RECOGNIZER_C_H_ +#define _FACE_RECOGNIZER_C_H_ +#include "../common/common.h" + +#ifdef __cplusplus +#include "recognizer.hpp" +extern "C" { +#endif + typedef void* IRecognizer; + IRecognizer new_mobilefacenet(); + void destroy_recognizer(IRecognizer r); + int recognizer_load_model(IRecognizer r, const char* root_path); + int extract_feature(IRecognizer r, const unsigned char* rgbdata, int img_width, int img_height, const Rect* face, FloatVector* feature); +#ifdef __cplusplus +} +#endif +#endif // !_FACE_RECOGNIZER_C_H_ diff --git a/src/face/recognizer/recognizer.hpp b/src/face/recognizer/recognizer.hpp new file mode 100644 index 0000000..f558af2 --- /dev/null +++ b/src/face/recognizer/recognizer.hpp @@ -0,0 +1,36 @@ +#ifndef _FACE_RECOGNIZER_H_ +#define _FACE_RECOGNIZER_H_ + +#include +#include "../common/common.hpp" + +namespace mirror { +class Recognizer { +public: + virtual ~Recognizer() {}; + virtual int LoadModel(const char* root_path) = 0; + virtual int ExtractFeature(const unsigned char* rgbdata, + int img_width, int img_height, + const Rect& face, + std::vector* feature) = 0; + +}; + +class RecognizerFactory { +public: + virtual Recognizer* CreateRecognizer() = 0; + virtual ~RecognizerFactory() {} + +}; + +class MobilefacenetRecognizerFactory : public RecognizerFactory { +public: + MobilefacenetRecognizerFactory() {}; + Recognizer* CreateRecognizer(); + ~MobilefacenetRecognizerFactory() {} +}; + +} + +#endif // !_FACE_RECOGNIZER_H_ + diff --git a/src/face/tracker/tracker.cpp b/src/face/tracker/tracker.cpp new file mode 100644 index 0000000..62e9e13 --- /dev/null +++ b/src/face/tracker/tracker.cpp @@ -0,0 +1,84 @@ +#include "tracker.h" +#include + +ITracker new_tracker() { + return new mirror::Tracker(); +} + +void destroy_tracker(ITracker t) { + delete static_cast(t); +} + +int track(ITracker t, const FaceInfoVector* curr_faces, TrackedFaceInfoVector* faces) { + std::vector cfaces; + for (int i = 0; i < curr_faces->length; ++i) { + cfaces.push_back(static_cast(curr_faces->faces[i])); + } + std::vector tfaces; + int ret = static_cast(t)->Track(cfaces, &tfaces); + if (ret != 0) { + return ret; + } + faces->length = tfaces.size(); + faces->faces = (TrackedFaceInfo*)malloc(faces->length * sizeof(TrackedFaceInfo)); + for (size_t i = 0; i < faces->length; ++i) { + faces->faces[i] = tfaces.at(i); + } + return 0; +} + +namespace mirror { +Tracker::Tracker() { + +} + +Tracker::~Tracker() { + +} + +int Tracker::Track(const std::vector& curr_faces, std::vector* faces) { + faces->clear(); + int num_faces = static_cast(curr_faces.size()); + + std::dequescored_tracked_faces(pre_tracked_faces_.begin(), pre_tracked_faces_.end()); + std::vector curr_tracked_faces; + for (int i = 0; i < num_faces; ++i) { + auto& face = curr_faces.at(i); + for (auto scored_tracked_face : scored_tracked_faces) { + ComputeIOU(scored_tracked_face.face_info_.location_, + face.location_, &scored_tracked_face.iou_score_); + } + if (scored_tracked_faces.size() > 0) { + std::partial_sort(scored_tracked_faces.begin(), + scored_tracked_faces.begin() + 1, + scored_tracked_faces.end(), + [](const TrackedFaceInfo &a, const TrackedFaceInfo &b) { + return a.iou_score_ > b.iou_score_; + }); + } + if (!scored_tracked_faces.empty() && scored_tracked_faces.front().iou_score_ > minScore_) { + TrackedFaceInfo matched_face = scored_tracked_faces.front(); + scored_tracked_faces.pop_front(); + TrackedFaceInfo &tracked_face = matched_face; + if (matched_face.iou_score_ < maxScore_) { + tracked_face.face_info_.location_.x = (tracked_face.face_info_.location_.x + face.location_.x) / 2; + tracked_face.face_info_.location_.y = (tracked_face.face_info_.location_.y + face.location_.y) / 2; + tracked_face.face_info_.location_.width = (tracked_face.face_info_.location_.width + face.location_.width) / 2; + tracked_face.face_info_.location_.height = (tracked_face.face_info_.location_.height + face.location_.height) / 2; + } else { + tracked_face.face_info_ = face; + } + curr_tracked_faces.push_back(tracked_face); + } else { + TrackedFaceInfo tracked_face; + tracked_face.face_info_ = face; + curr_tracked_faces.push_back(tracked_face); + } + } + + pre_tracked_faces_ = curr_tracked_faces; + *faces = curr_tracked_faces; + return 0; +} + +} diff --git a/src/face/tracker/tracker.h b/src/face/tracker/tracker.h new file mode 100644 index 0000000..7af7118 --- /dev/null +++ b/src/face/tracker/tracker.h @@ -0,0 +1,16 @@ +#ifndef _FACE_TRACKER_C_H_ +#define _FACE_TRACKER_C_H_ +#include "../common/common.h" + +#ifdef __cplusplus +#include "tracker.hpp" +extern "C" { +#endif + typedef void* ITracker; + ITracker new_tracker(); + void destroy_tracker(ITracker t); + int track(ITracker t, const FaceInfoVector* curr_faces, TrackedFaceInfoVector* faces); +#ifdef __cplusplus +} +#endif +#endif // !_FACE_TRACKER_C_H_ diff --git a/src/face/tracker/tracker.hpp b/src/face/tracker/tracker.hpp new file mode 100644 index 0000000..c682bf2 --- /dev/null +++ b/src/face/tracker/tracker.hpp @@ -0,0 +1,23 @@ +#ifndef _FACE_TRACKER_H_ +#define _FACE_TRACKER_H_ + +#include +#include "../common/common.h" + +namespace mirror { +class Tracker { +public: + Tracker(); + ~Tracker(); + int Track(const std::vector& curr_faces, + std::vector* faces); + +private: + std::vector pre_tracked_faces_; + const float minScore_ = 0.3f; + const float maxScore_ = 0.5f; +}; + +} + +#endif // !_FACE_TRACKER_H_