zishuo/openvision
1
0
Fork 0
mirror of https://github.com/bububa/openvision.git synced 2025-09-26 17:51:13 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(counter): add p2pnet crowd counter

Browse Source
This commit is contained in:
Syd Xu
2021-11-11 15:35:49 +08:00
parent f60faaa7ed
commit 9ffdeb3f3c
15 changed files with 482 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -54,6 +54,8 @@ cmake .. # optional -DNCNN_VULKAN=OFF -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COM
- animegan2 [Google Drive](https://drive.google.com/drive/folders/1K6ZScENPHVbxupHkwl5WcpG8PPECtD8e?usp=sharing)
- tracker
- lighttrack [Google Drive](https://drive.google.com/drive/folders/16cxns_xzSOABHn6UcY1OXyf4MFcSSbEf?usp=sharing)
- counter
- p2pnet [Google Drive](https://drive.google.com/drive/folders/1kmtBsPIS79C3hMAwm_Tv9tAPvJLV9k35?usp=sharing)
- golang binding (github.com/bububa/openvision/go)
## Reference

2
go/README.md
View File

@@ -66,3 +66,5 @@ make -j 4
- animegan2 [Google Drive](https://drive.google.com/drive/folders/1K6ZScENPHVbxupHkwl5WcpG8PPECtD8e?usp=sharing)
- tracker
- lighttrack [Google Drive](https://drive.google.com/drive/folders/16cxns_xzSOABHn6UcY1OXyf4MFcSSbEf?usp=sharing)
- counter
- p2pnet [Google Drive](https://drive.google.com/drive/folders/1kmtBsPIS79C3hMAwm_Tv9tAPvJLV9k35?usp=sharing)

12
go/counter/cgo.go Normal file
View File

@@ -0,0 +1,12 @@
//go:build !vulkan
// +build !vulkan
package counter
/*
#cgo CXXFLAGS: --std=c++11 -fopenmp
#cgo CPPFLAGS: -I ${SRCDIR}/../../include -I /usr/local/include
#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision
#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../lib
*/
import "C"

11
go/counter/cgo_vulkan.go Normal file
View File

@@ -0,0 +1,11 @@
// +build vulkan
package counter
/*
#cgo CXXFLAGS: --std=c++11 -fopenmp
#cgo CPPFLAGS: -I ${SRCDIR}/../../include -I /usr/local/include
#cgo LDFLAGS: -lstdc++ -lncnn -lomp -lopenvision -lglslang -lvulkan -lSPIRV -lOGLCompiler -lMachineIndependent -lGenericCodeGen -lOSDependent
#cgo LDFLAGS: -L /usr/local/lib -L ${SRCDIR}/../../lib
*/
import "C"

40
go/counter/counter.go Normal file
View File

@@ -0,0 +1,40 @@
package counter
/*
#include <stdlib.h>
#include <stdbool.h>
#include "openvision/common/common.h"
#include "openvision/counter/counter.h"
*/
import "C"
import (
"unsafe"
openvision "github.com/bububa/openvision/go"
"github.com/bububa/openvision/go/common"
)
// Counter represents Object Counter interface
type Counter interface {
common.Estimator
CrowdCount(img *common.Image) ([]common.Keypoint, error)
}
// CrowdCount returns object counter
func CrowdCount(d Counter, img *common.Image) ([]common.Keypoint, error) {
imgWidth := img.WidthF64()
imgHeight := img.HeightF64()
data := img.Bytes()
ptsC := common.NewCKeypointVector()
defer common.FreeCKeypointVector(ptsC)
errCode := C.crowd_count(
(C.ICounter)(d.Pointer()),
(*C.uchar)(unsafe.Pointer(&data[0])),
C.int(imgWidth),
C.int(imgHeight),
(*C.KeypointVector)(unsafe.Pointer(ptsC)))
if errCode != 0 {
return nil, openvision.CounterError(int(errCode))
}
return common.GoKeypointVector(ptsC, imgWidth, imgHeight), nil
}

2
go/counter/doc.go Normal file
View File

@@ -0,0 +1,2 @@
// Package counter include object counter
package counter

45
go/counter/p2pnet.go Normal file
View File

@@ -0,0 +1,45 @@
package counter
/*
#include <stdlib.h>
#include <stdbool.h>
#include "openvision/counter/counter.h"
*/
import "C"
import (
"unsafe"
"github.com/bububa/openvision/go/common"
)
// P2PNet represents p2pnet counter
type P2PNet struct {
d C.ICounter
}
// NewP2PNet returns a new P2PNet
func NewP2PNet() *P2PNet {
return &P2PNet{
d: C.new_p2pnet_crowd_counter(),
}
}
// Destroy free tracker
func (d *P2PNet) Destroy() {
common.DestroyEstimator(d)
}
// Pointer implement Estimator interface
func (d *P2PNet) Pointer() unsafe.Pointer {
return unsafe.Pointer(d.d)
}
// LoadModel load model for detecter
func (d *P2PNet) LoadModel(modelPath string) error {
return common.EstimatorLoadModel(d, modelPath)
}
// CrowdCount implement Object Counter interface
func (d *P2PNet) CrowdCount(img *common.Image) ([]common.Keypoint, error) {
return CrowdCount(d, img)
}

6
go/error.go
View File

@@ -74,6 +74,12 @@ var (
Message: "object tracker error",
}
}
CounterError = func(code int) Error {
return Error{
Code: code,
Message: "object counter error",
}
}
RealsrError = func(code int) Error {
return Error{
Code: code,

92
go/examples/counter/main.go Normal file
View File

@@ -0,0 +1,92 @@
package main
import (
"bytes"
"image"
"image/jpeg"
"log"
"os"
"os/user"
"path/filepath"
"strings"
"github.com/bububa/openvision/go/common"
"github.com/bububa/openvision/go/counter"
)
func main() {
wd, _ := os.Getwd()
dataPath := cleanPath(wd, "~/go/src/github.com/bububa/openvision/data")
imgPath := filepath.Join(dataPath, "./images")
modelPath := filepath.Join(dataPath, "./models")
common.CreateGPUInstance()
defer common.DestroyGPUInstance()
cpuCores := common.GetBigCPUCount()
common.SetOMPThreads(cpuCores)
log.Printf("CPU big cores:%d\n", cpuCores)
d := p2pnet(modelPath)
defer d.Destroy()
common.SetEstimatorThreads(d, cpuCores)
crowdCount(d, imgPath, "congested2.jpg")
}
func p2pnet(modelPath string) counter.Counter {
modelPath = filepath.Join(modelPath, "p2pnet")
d := counter.NewP2PNet()
if err := d.LoadModel(modelPath); err != nil {
log.Fatalln(err)
}
return d
}
func crowdCount(d counter.Counter, imgPath string, filename string) {
inPath := filepath.Join(imgPath, filename)
imgLoaded, err := loadImage(inPath)
if err != nil {
log.Fatalln("load image failed,", err)
}
img := common.NewImage(imgLoaded)
pts, err := d.CrowdCount(img)
if err != nil {
log.Fatalln(err)
}
log.Printf("count: %d\n", len(pts))
}
func loadImage(filePath string) (image.Image, error) {
fn, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer fn.Close()
img, _, err := image.Decode(fn)
if err != nil {
return nil, err
}
return img, nil
}
func saveImage(img image.Image, filePath string) error {
buf := new(bytes.Buffer)
if err := jpeg.Encode(buf, img, nil); err != nil {
return err
}
fn, err := os.Create(filePath)
if err != nil {
return err
}
defer fn.Close()
fn.Write(buf.Bytes())
return nil
}
func cleanPath(wd string, path string) string {
usr, _ := user.Current()
dir := usr.HomeDir
if path == "~" {
return dir
} else if strings.HasPrefix(path, "~/") {
return filepath.Join(dir, path[2:])
}
return filepath.Join(wd, path)
}

6
src/CMakeLists.txt
View File

@@ -73,6 +73,8 @@ target_include_directories(openvision
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/styletransfer>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/tracker>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/counter>
)
#install(TARGETS openvision EXPORT openvision ARCHIVE DESTINATION ${LIBRARY_OUTPUT_PATH})
@@ -115,3 +117,7 @@ file(COPY
DESTINATION ${INCLUDE_OUTPUT_PATH}/openvision/tracker
)
file(COPY
${CMAKE_CURRENT_SOURCE_DIR}/counter/counter.h
DESTINATION ${INCLUDE_OUTPUT_PATH}/openvision/counter
)

21
src/counter/counter.cpp Normal file
View File

@@ -0,0 +1,21 @@
#include "counter.h"
#include "p2pnet/p2pnet.hpp"
ICounter new_p2pnet_crowd_counter() { return new ovcounter::P2PNet(); }
int crowd_count(ICounter d, const unsigned char *rgbdata, int img_width,
int img_height, KeypointVector *keypoints) {
std::vector<ov::Keypoint> pts;
int ret = static_cast<ovcounter::P2PNet *>(d)->CrowdCount(rgbdata, img_width,
img_height, &pts);
if (ret != 0) {
return ret;
}
keypoints->length = pts.size();
keypoints->points =
(ov::Keypoint *)malloc(keypoints->length * sizeof(ov::Keypoint));
for (size_t i = 0; i < keypoints->length; ++i) {
keypoints->points[i] = pts.at(i);
}
return 0;
}

17
src/counter/counter.h Normal file
View File

@@ -0,0 +1,17 @@
#ifndef _COUNTER_C_H_
#define _COUNTER_C_H_
#include "../common/common.h"
#ifdef __cplusplus
#include "counter.hpp"
extern "C" {
#endif
typedef void *ICounter;
ICounter new_p2pnet_crowd_counter();
int crowd_count(ICounter d, const unsigned char *rgbdata, int img_width,
int img_height, KeypointVector *pts);
#ifdef __cplusplus
}
#endif
#endif // !_COUNTER_C_H_

13
src/counter/counter.hpp Normal file
View File

@@ -0,0 +1,13 @@
#ifndef _COUNTER_H_
#define _COUNTER_H_
#include "../common/common.hpp"
namespace ovcounter {
class Counter : public ov::Estimator {
public:
virtual int CrowdCount(const unsigned char *rgbdata, int img_width,
int img_height, std::vector<ov::Keypoint> *pts) = 0;
};
} // namespace ovcounter
#endif // !_COUNTER_H_

185
src/counter/p2pnet/p2pnet.cpp Normal file
View File

@@ -0,0 +1,185 @@
#include "p2pnet.hpp"
#include <math.h>
#ifdef OV_VULKAN
#include "gpu.h"
#endif // OV_VULKAN
namespace ovcounter {
void P2PNet::shift(int w, int h, int stride, std::vector<float> anchor_points,
std::vector<float> &shifted_anchor_points) {
std::vector<float> x_, y_;
for (int i = 0; i < w; i++) {
float x = (i + 0.5) * stride;
x_.push_back(x);
}
for (int i = 0; i < h; i++) {
float y = (i + 0.5) * stride;
y_.push_back(y);
}
std::vector<float> shift_x(w * h, 0), shift_y(w * h, 0);
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j++) {
shift_x[i * w + j] = x_[j];
}
}
for (int i = 0; i < h; i++) {
for (int j = 0; j < w; j++) {
shift_y[i * w + j] = y_[i];
}
}
std::vector<float> shifts(w * h * 2, 0);
for (int i = 0; i < w * h; i++) {
shifts[i * 2] = shift_x[i];
shifts[i * 2 + 1] = shift_y[i];
}
shifted_anchor_points.resize(2 * w * h * anchor_points.size() / 2, 0);
for (int i = 0; i < w * h; i++) {
for (int j = 0; j < anchor_points.size() / 2; j++) {
float x = anchor_points[j * 2] + shifts[i * 2];
float y = anchor_points[j * 2 + 1] + shifts[i * 2 + 1];
shifted_anchor_points[i * anchor_points.size() / 2 * 2 + j * 2] = x;
shifted_anchor_points[i * anchor_points.size() / 2 * 2 + j * 2 + 1] = y;
}
}
}
void P2PNet::generate_anchor_points(int stride, int row, int line,
std::vector<float> &anchor_points) {
float row_step = (float)stride / row;
float line_step = (float)stride / line;
std::vector<float> x_, y_;
for (int i = 1; i < line + 1; i++) {
float x = (i - 0.5) * line_step - (float)stride / 2;
x_.push_back(x);
}
for (int i = 1; i < row + 1; i++) {
float y = (i - 0.5) * row_step - (float)stride / 2;
y_.push_back(y);
}
std::vector<float> shift_x(row * line, 0), shift_y(row * line, 0);
for (int i = 0; i < row; i++) {
for (int j = 0; j < line; j++) {
shift_x[i * line + j] = x_[j];
}
}
for (int i = 0; i < row; i++) {
for (int j = 0; j < line; j++) {
shift_y[i * line + j] = y_[i];
}
}
anchor_points.resize(row * line * 2, 0);
for (int i = 0; i < row * line; i++) {
float x = shift_x[i];
float y = shift_y[i];
anchor_points[i * 2] = x;
anchor_points[i * 2 + 1] = y;
}
}
void P2PNet::generate_anchor_points(int img_w, int img_h,
std::vector<int> pyramid_levels, int row,
int line,
std::vector<float> &all_anchor_points) {
std::vector<std::pair<int, int>> image_shapes;
std::vector<int> strides;
for (int i = 0; i < pyramid_levels.size(); i++) {
int new_h = floor((img_h + pow(2, pyramid_levels[i]) - 1) /
pow(2, pyramid_levels[i]));
int new_w = floor((img_w + pow(2, pyramid_levels[i]) - 1) /
pow(2, pyramid_levels[i]));
image_shapes.push_back(std::make_pair(new_w, new_h));
strides.push_back(pow(2, pyramid_levels[i]));
}
all_anchor_points.clear();
for (int i = 0; i < pyramid_levels.size(); i++) {
std::vector<float> anchor_points;
generate_anchor_points(pow(2, pyramid_levels[i]), row, line, anchor_points);
std::vector<float> shifted_anchor_points;
shift(image_shapes[i].first, image_shapes[i].second, strides[i],
anchor_points, shifted_anchor_points);
all_anchor_points.insert(all_anchor_points.end(),
shifted_anchor_points.begin(),
shifted_anchor_points.end());
}
}
int P2PNet::CrowdCount(const unsigned char *rgbdata, int img_width,
int img_height, std::vector<ov::Keypoint> *keypoints) {
if (!initialized_) {
return 10000;
}
if (rgbdata == 0) {
return 10001;
}
// pad to multiple of 32
int w = img_width;
int h = img_height;
float scale = 1.f;
if (w > h) {
scale = (float)target_size / w;
w = target_size;
h = h * scale;
} else {
scale = (float)target_size / h;
h = target_size;
w = w * scale;
}
ncnn::Mat input = ncnn::Mat::from_pixels_resize(rgbdata, ncnn::Mat::PIXEL_RGB,
img_width, img_height, w, h);
// pad to target_size rectangle
int wpad = (w + 31) / 32 * 32 - w;
int hpad = (h + 31) / 32 * 32 - h;
ncnn::Mat in_pad;
ncnn::copy_make_border(input, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2,
wpad - wpad / 2, ncnn::BORDER_CONSTANT, 0.f);
std::vector<int> pyramid_levels(1, 3);
std::vector<float> all_anchor_points;
generate_anchor_points(in_pad.w, in_pad.h, pyramid_levels, 2, 2,
all_anchor_points);
ncnn::Mat anchor_points =
ncnn::Mat(2, all_anchor_points.size() / 2, all_anchor_points.data());
ncnn::Extractor ex = net_->create_extractor();
ex.set_light_mode(light_mode_);
ex.set_num_threads(num_threads);
in_pad.substract_mean_normalize(mean_vals1, norm_vals1);
ex.input("input", in_pad);
ex.input("anchor", anchor_points);
ncnn::Mat score, points;
ex.extract("pred_scores", score);
ex.extract("pred_points", points);
keypoints->clear();
for (int i = 0; i < points.h; i++) {
float *score_data = score.row(i);
float *points_data = points.row(i);
if (score_data[1] > 0.5) {
ov::Keypoint kpt;
int x = (points_data[0] - ((float)wpad / 2)) / scale;
int y = (points_data[1] - ((float)hpad / 2)) / scale;
kpt.p = ov::Point2f(x, y);
kpt.score = score_data[1];
keypoints->push_back(kpt);
}
}
return 0;
}
} // namespace ovcounter

28
src/counter/p2pnet/p2pnet.hpp Normal file
View File

@@ -0,0 +1,28 @@
#ifndef _COUNTER_P2PNET_H_
#define _COUNTER_P2PNET_H_
#include "../counter.hpp"
namespace ovcounter {
class P2PNet : public Counter {
public:
int CrowdCount(const unsigned char *rgbdata, int img_width, int img_height,
std::vector<ov::Keypoint> *keypoints);
private:
const int target_size = 640;
const float mean_vals1[3] = {123.675f, 116.28f, 103.53f};
const float norm_vals1[3] = {0.01712475f, 0.0175f, 0.01742919f};
void generate_anchor_points(int stride, int row, int line,
std::vector<float> &anchor_points);
void generate_anchor_points(int img_w, int img_h,
std::vector<int> pyramid_levels, int row,
int line, std::vector<float> &all_anchor_points);
void shift(int w, int h, int stride, std::vector<float> anchor_points,
std::vector<float> &shifted_anchor_points);
};
} // namespace ovcounter
#endif // !_COUNTER_P2PNET_H_