[Other] Refactor js submodule (#415)

* Refactor js submodule

* Remove change-log

* Update ocr module

* Update ocr-detection module

* Update ocr-detection module

* Remove change-log
This commit is contained in:
chenqianhe
2022-10-23 14:05:13 +08:00
committed by GitHub
parent 30971cf3fd
commit f2619b0546
273 changed files with 14697 additions and 5088 deletions

View File

@@ -0,0 +1,10 @@
{
"presets": [
[
"@babel/preset-env",
{
"modules": false
}
]
]
}

View File

@@ -0,0 +1,3 @@
module.exports = {
extends: ['@commitlint/config-conventional'], // 使用预设的配置 https://github.com/conventional-changelog/commitlint/blob/master/@commitlint/config-conventional/index.js
}

View File

@@ -0,0 +1 @@
node_modules

View File

@@ -0,0 +1,27 @@
module.exports = {
parser: '@typescript-eslint/parser', // 使用 ts 解析器
extends: [
'eslint:recommended', // eslint 推荐规则
'plugin:@typescript-eslint/recommended', // ts 推荐规则
'plugin:jest/recommended',
],
plugins: [
'@typescript-eslint',
'jest',
],
env: {
browser: true,
node: true,
es6: true,
},
parserOptions: {
project: 'tsconfig.eslint.json',
ecmaVersion: 2019,
sourceType: 'module',
ecmaFeatures: {
experimentalObjectRestSpread: true
}
},
rules: {}
}

View File

@@ -0,0 +1,30 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
.DS_Store
dist
etc
lib
dist-ssr
coverage
*.local
/cypress/videos/
/cypress/screenshots/
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?

View File

@@ -0,0 +1 @@
auto-install-peers=true

View File

@@ -0,0 +1,28 @@
[中文版](./README_cn.md)
# gesture
gesture is a gesture recognition module, including two models: gesture_detect and gesture_rec. gesture_detect model is used to identify the palm area of the person in the image. gesture_rec model is used to recognize human gesture. The interface provided by the module is simple, users only need to pass in gesture images to get the results.
<img src="https://img.shields.io/npm/v/@paddle-js-models/gesture?color=success" alt="version"> <img src="https://img.shields.io/bundlephobia/min/@paddle-js-models/gesture" alt="size"> <img src="https://img.shields.io/npm/dm/@paddle-js-models/gesture?color=orange" alt="downloads"> <img src="https://img.shields.io/npm/dt/@paddle-js-models/gesture" alt="downloads">
# Usage
```js
import * as gesture from '@paddle-js-models/gesture';
// Load gesture_detect model and gesture_rec model
await gesture.load();
// Get the image recognition results. The results include: palm frame coordinates and recognition results
const res = await gesture.classify(img);
```
# Online experience
https://paddlejs.baidu.com/gesture
# Performance
<img alt="gesture" src="https://user-images.githubusercontent.com/43414102/156379706-065a4f57-cc75-4457-857a-18619589492f.gif">

View File

@@ -0,0 +1,28 @@
[English](./README.md)
# gesture
gesture 为手势识别模块包括两个模型gesture_detect和gesture_rec。gesture_detect模型识别图片中人物手掌区域gesture_rec模型识别人物手势。模块提供的接口简单使用者只需传入手势图片即可获得结果。
<img src="https://img.shields.io/npm/v/@paddle-js-models/gesture?color=success" alt="version"> <img src="https://img.shields.io/bundlephobia/min/@paddle-js-models/gesture" alt="size"> <img src="https://img.shields.io/npm/dm/@paddle-js-models/gesture?color=orange" alt="downloads"> <img src="https://img.shields.io/npm/dt/@paddle-js-models/gesture" alt="downloads">
# 使用
```js
import * as gesture from '@paddle-js-models/gesture';
// gesture_detect、gesture_rec模型加载
await gesture.load();
// 获取图片识别结果。结果包括:手掌框选坐标和识别结果
const res = await gesture.classify(img);
```
# 在线体验
https://paddlejs.baidu.com/gesture
# 效果
<img alt="gesture" src="https://user-images.githubusercontent.com/43414102/156379706-065a4f57-cc75-4457-857a-18619589492f.gif">

View File

@@ -0,0 +1,15 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
"mainEntryPointFilePath": "./lib/index.d.ts",
"bundledPackages": [],
"docModel": {
"enabled": true
},
"apiReport": {
"enabled": true
},
"dtsRollup": {
"enabled": true,
"untrimmedFilePath": "./lib/index.d.ts"
}
}

View File

@@ -0,0 +1,17 @@
import path from "path";
import chalk from "chalk";
export const paths = {
root: path.join(__dirname, '../'),
input: path.join(__dirname, '../src/index.ts'),
lib: path.join(__dirname, '../lib'),
}
export const log = {
progress: (text: string) => {
console.log(chalk.green(text))
},
error: (text: string) => {
console.log(chalk.red(text))
},
}

View File

@@ -0,0 +1,160 @@
import path from 'path'
import fse from 'fs-extra'
import { series } from "gulp"
import { paths, log } from "./build_package/util"
import rollupConfig from './rollup.config'
import { rollup } from 'rollup'
import {
Extractor,
ExtractorConfig,
ExtractorResult,
} from '@microsoft/api-extractor'
/**
* 这里是由于 'conventional-changelog' 未提供类型文件
*/
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import conventionalChangelog from 'conventional-changelog'
interface TaskFunc {
// eslint-disable-next-line @typescript-eslint/ban-types
(cb: Function): void
}
const CHANGE_TRACE = ['paddlejs-models/gesture', 'paddle-js-models/gesture', 'paddlejs-models', 'paddle-js-models', 'all']
/**
* 删除 lib 文件
* @param cb
* @returns {Promise<void>}
*/
const clearLibFile: TaskFunc = async (cb) => {
fse.removeSync(paths.lib)
log.progress('Deleted lib file')
cb()
}
/**
* rollup 打包
* @param cb
*/
const buildByRollup: TaskFunc = async (cb) => {
const inputOptions = {
input: rollupConfig.input,
external: rollupConfig.external,
plugins: rollupConfig.plugins,
}
const outOptions = rollupConfig.output
const bundle = await rollup(inputOptions)
// 写入需要遍历输出配置
if (Array.isArray(outOptions)) {
for (const outOption of outOptions) {
await bundle.write(outOption)
}
cb()
log.progress('Rollup built successfully')
}
}
/**
* api-extractor 整理 .d.ts 文件
* @param cb
*/
const apiExtractorGenerate: TaskFunc = async (cb) => {
const apiExtractorJsonPath: string = path.join(__dirname, './api-extractor.json')
// 加载并解析 api-extractor.json 文件
const extractorConfig: ExtractorConfig = await ExtractorConfig.loadFileAndPrepare(apiExtractorJsonPath)
// 判断是否存在 index.d.ts 文件,这里必须异步先访问一边,不然后面找不到会报错
const isdtxExist: boolean = await fse.pathExists(extractorConfig.mainEntryPointFilePath)
// 判断是否存在 etc 目录api-extractor需要该目录存在
const isEtcExist: boolean = await fse.pathExists('./etc')
if (!isdtxExist) {
log.error('API Extractor not find index.d.ts')
return
}
if (!isEtcExist) {
fse.mkdirSync('etc');
log.progress('Create folder etc for API Extractor')
}
// 调用 API
const extractorResult: ExtractorResult = await Extractor.invoke(extractorConfig, {
localBuild: true,
// 在输出中显示信息
showVerboseMessages: true,
})
if (extractorResult.succeeded) {
// 删除多余的 .d.ts 文件
const libFiles: string[] = await fse.readdir(paths.lib)
for (const file of libFiles) {
if (file.endsWith('.d.ts') && !file.includes('index')) {
await fse.remove(path.join(paths.lib, file))
}
}
log.progress('API Extractor completed successfully')
// api-extractor 会生成 temp 文件夹,完成后进行删除
fse.ensureDirSync('temp')
fse.removeSync('temp')
cb()
} else {
log.error(`API Extractor completed with ${extractorResult.errorCount} errors`
+ ` and ${extractorResult.warningCount} warnings`)
}
}
/**
* 完成
* @param cb
*/
const complete: TaskFunc = (cb) => {
log.progress('---- end ----')
cb()
}
/**
* 生成 CHANGELOG
* @param cb
*/
export const changelog: TaskFunc = async (cb) => {
const checkTrace = (chunk: string) => {
for (const keyWord of CHANGE_TRACE) {
if (chunk.includes(keyWord)) {
return true
}
}
return false
}
const changelogPath: string = path.join(paths.root, 'CHANGELOG.md')
// 对命令 conventional-changelog -p angular -i CHANGELOG.md -w -r 0
const changelogPipe = await conventionalChangelog({
preset: 'angular',
releaseCount: 0,
})
changelogPipe.setEncoding('utf8')
const resultArray = ['# 更新日志\n\n']
changelogPipe.on('data', (chunk) => {
// 原来的 commits 路径是进入提交列表
chunk = chunk.replace(/\/commits\//g, '/commit/')
/**
* title 或 指定跟踪 才会写入CHANGELOG
*/
for (const log of chunk.split("\n")) {
if (log.includes('# ') || log.includes('### ') || checkTrace(log)) {
resultArray.push(log+"\n\n")
}
}
})
changelogPipe.on('end', async () => {
fse.createWriteStream(changelogPath).write(resultArray.join(''))
cb()
log.progress('CHANGELOG generation completed')
})
}
exports.build = series(clearLibFile, buildByRollup, apiExtractorGenerate, complete)

View File

@@ -0,0 +1,4 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
}

View File

@@ -0,0 +1,77 @@
{
"name": "@paddle-js-models/gesture",
"version": "3.0.1",
"description": "",
"main": "lib/index.js",
"module": "lib/index.esm.js",
"typings": "lib/index.d.js",
"files": [
"lib",
"LICENSE",
"CHANGELOG.md",
"README.md",
"README_cn.md"
],
"keywords": [],
"author": "",
"license": "ISC",
"publishConfig": {
"access": "public",
"registry": "https://registry.npmjs.org/"
},
"scripts": {
"dev": "yalc publish --push",
"prepublish": "pnpm lint & pnpm test",
"prepublishOnly": "pnpm build",
"build": "gulp build",
"lint": "eslint --ext .js,.ts src --fix",
"api": "api-extractor run",
"test": "jest --coverage --verbose -u",
"changelog": "gulp changelog"
},
"devDependencies": {
"@babel/core": "^7.19.0",
"@babel/preset-env": "^7.19.0",
"@commitlint/cli": "^17.1.2",
"@commitlint/config-conventional": "^17.1.0",
"@microsoft/api-extractor": "^7.30.0",
"@types/d3-polygon": "^3.0.0",
"@types/fs-extra": "^9.0.13",
"@types/gulp": "^4.0.9",
"@types/jest": "^29.0.1",
"@types/node": "^18.7.16",
"@typescript-eslint/eslint-plugin": "^5.36.2",
"@typescript-eslint/parser": "^5.36.2",
"browserify": "^17.0.0",
"chalk": "4.1.2",
"commitlint": "^17.1.2",
"conventional-changelog-cli": "^2.2.2",
"eslint": "8.22.0",
"eslint-plugin-jest": "^27.0.4",
"fs-extra": "^10.1.0",
"gulp": "^4.0.2",
"gulp-clean": "^0.4.0",
"gulp-typescript": "6.0.0-alpha.1",
"gulp-uglify": "^3.0.2",
"husky": "^8.0.1",
"jest": "^29.0.3",
"lint-staged": "^13.0.3",
"rollup": "^2.79.0",
"rollup-plugin-babel": "^4.4.0",
"rollup-plugin-commonjs": "^10.1.0",
"rollup-plugin-eslint": "^7.0.0",
"rollup-plugin-node-resolve": "^5.2.0",
"rollup-plugin-string": "^3.0.0",
"rollup-plugin-typescript2": "^0.34.0",
"ts-jest": "^29.0.0",
"ts-node": "^10.9.1",
"tsify": "^5.0.4",
"typescript": "^4.8.3",
"vinyl-buffer": "^1.0.1",
"vinyl-source-stream": "^2.0.0"
},
"dependencies": {
"@paddlejs/paddlejs-backend-webgl": "^1.2.9",
"@paddlejs/paddlejs-core": "^2.2.0"
}
}

View File

@@ -0,0 +1,76 @@
import path from 'path'
import { RollupOptions } from 'rollup'
import { string } from "rollup-plugin-string";
import rollupTypescript from 'rollup-plugin-typescript2'
import babel from 'rollup-plugin-babel'
import resolve from 'rollup-plugin-node-resolve'
import commonjs from 'rollup-plugin-commonjs'
import { eslint } from 'rollup-plugin-eslint'
import { DEFAULT_EXTENSIONS } from '@babel/core'
import pkg from './package.json'
import { paths } from "./build_package/util";
// rollup 配置项
const rollupConfig: RollupOptions = {
input: paths.input,
output: [
// 输出 commonjs 规范的代码
{
file: path.join(paths.lib, 'index.js'),
format: 'cjs',
name: pkg.name,
},
// 输出 es 规范的代码
{
file: path.join(paths.lib, 'index.esm.js'),
format: 'es',
name: pkg.name,
},
],
external: ['@paddlejs-mediapipe/opencv',
'@paddlejs/paddlejs-backend-webgl',
'@paddlejs/paddlejs-core',
'@types/node',
'd3-polygon',
'js-clipper',
'number-precision'],
// plugins 需要注意引用顺序
plugins: [
eslint({
throwOnError: true,
throwOnWarning: false,
include: ['src/**/*.ts'],
exclude: ['node_modules/**', 'lib/**', '*.js'],
}),
// 处理txt文件
string({
include: "src/anchor_small.txt"
}),
// 使得 rollup 支持 commonjs 规范,识别 commonjs 规范的依赖
commonjs(),
// 配合 commnjs 解析第三方模块
resolve({
// 将自定义选项传递给解析插件
customResolveOptions: {
moduleDirectory: 'node_modules',
},
}),
rollupTypescript(),
babel({
runtimeHelpers: true,
// 只转换源代码,不运行外部依赖
exclude: 'node_modules/**',
// babel 默认不支持 ts 需要手动添加
extensions: [
...DEFAULT_EXTENSIONS,
'.ts',
],
}),
],
}
export default rollupConfig

View File

@@ -0,0 +1,100 @@
export default class LMProcess {
private input_n: number;
private input_h: number;
private input_w: number;
private input_c: number;
private class_dim: number;
private points_num: number;
private result: any;
private conf!: number;
private kp: any;
private forefinger: any;
private output_softmax: any;
type!: string;
constructor(result) {
/*
* result[0] :是否为手
* result[1:43]: 手的21个关键点
* result[43:49]: 几中手势分类,包含但不限于石头剪刀布,为了提升准确度
* this.kp decode result得出的21个手指关键点this.kp[8]为食指
* this.conf 是否为手,越大,越可能是手
*/
this.input_n = 1;
this.input_h = 224;
this.input_w = 224;
this.input_c = 3;
this.class_dim = 6;
this.points_num = 1;
this.result = result;
}
sigm(value) {
return 1.0 / (1.0 + Math.exp(0.0 - value));
}
decodeConf() {
this.conf = this.sigm(this.result[0]);
}
decodeKp() {
// 21个关键点下标1开始
const offset = 1;
const result = this.result;
this.kp = [];
for (let i = 0; i < this.points_num; i++) {
const arr: number[] = [];
arr.push((result[offset + i * 2] + 0.5) * this.input_h);
arr.push((result[offset + i * 2 + 1] + 0.5) * this.input_h);
this.kp.push(arr);
}
this.forefinger = this.kp[0];
}
softMax() {
let max = 0;
let sum = 0;
const offset = 2;
const class_dim = this.class_dim = 7;
const result = this.result;
const output_softmax = new Array(7).fill(null);
for (let i = 0; i < class_dim; i++) {
if (max < result[i + offset]) {
max = result[i + offset];
}
}
for (let i = 0; i < class_dim; i++) {
output_softmax[i] = Math.exp(result[i + offset] - max);
sum += output_softmax[i];
}
for (let i = 0; i < class_dim; i++) {
output_softmax[i] /= sum;
}
this.output_softmax = output_softmax;
}
output() {
this.decodeKp();
this.softMax();
let label_index = 0;
let max_pro = this.output_softmax[0];
for (let i = 1; i < this.class_dim; i++) {
if (max_pro < this.output_softmax[i]) {
label_index = i;
max_pro = this.output_softmax[i];
}
}
// 最后一位:有无手
if (label_index !== 0 && label_index !== this.class_dim - 1 && max_pro > 0.9) {
const ges = ['其他', '布', '剪刀', '石头', '1', 'ok'];
this.type = ges[label_index];
return;
}
this.type = 'other';
}
}

View File

@@ -0,0 +1,414 @@
import WarpAffine from './warpAffine';
class DetectProcess {
private modelResult: any;
private output_size: number;
private anchor_num: number;
private detectResult: any;
private hasHand: number;
private originCanvas: HTMLCanvasElement;
private originImageData: ImageData;
private maxIndex!: number;
private box: any;
private feed: any;
private anchors: any;
private mtr: any;
private source!: any[][];
private imageData!: ImageData;
private pxmin: any;
private pxmax: any;
private pymin: any;
private pymax: any;
private kp!: number[][];
private tw!: number;
private th!: number;
private triangle: any;
private results: any;
constructor(result, canvas) {
this.modelResult = result;
this.output_size = 10;
this.anchor_num = 1920;
this.detectResult = new Array(14).fill('').map(() => []);
this.hasHand = 0;
this.originCanvas = canvas;
const ctx = this.originCanvas.getContext('2d');
this.originImageData = ctx!.getImageData(0, 0, 256, 256);
}
async outputBox(results) {
this.getMaxIndex();
if (this.maxIndex === -1) {
this.hasHand = 0;
return false;
}
this.hasHand = 1;
await this.splitAnchors(results);
this.decodeAnchor();
// 求关键点
this.decodeKp();
// 求手势框
this.decodeBox();
return this.box;
}
async outputFeed(paddle) {
this.decodeTriangle();
this.decodeSource();
this.outputResult();
// 提取仿射变化矩阵
this.getaffinetransform();
// 对图片进行仿射变换
await this.warpAffine();
this.allReshapeToRGB(paddle);
return this.feed;
}
async splitAnchors(results) {
this.results = results;
const anchors: number[][] = new Array(this.anchor_num).fill('').map(() => []);
const anchor_num = this.anchor_num;
for (let i = 0; i < anchor_num; i++) {
const tmp0 = results[i * 4];
const tmp1 = results[i * 4 + 1];
const tmp2 = results[i * 4 + 2];
const tmp3 = results[i * 4 + 3];
anchors[i] = [
tmp0 - tmp2 / 2.0,
tmp1 - tmp3 / 2.0,
tmp0 + tmp2 / 2.0,
tmp1 + tmp3 / 2.0
];
if (anchors[i][0] < 0) {
anchors[i][0] = 0;
}
if (anchors[i][0] > 1) {
anchors[i][0] = 1;
}
if (anchors[i][1] < 0) {
anchors[i][1] = 0;
}
if (anchors[i][1] > 1) {
anchors[i][1] = 1;
}
if (anchors[i][2] < 0) {
anchors[i][2] = 0;
}
if (anchors[i][2] > 1) {
anchors[i][2] = 1;
}
if (anchors[i][3] < 0) {
anchors[i][3] = 0;
}
if (anchors[i][3] > 1) {
anchors[i][3] = 1;
}
}
this.anchors = anchors;
}
getMaxIndex() {
let maxIndex = -1;
let maxConf = -1;
let curConf = -2.0;
const output_size = 10;
for (let i = 0; i < this.anchor_num; i++) {
curConf = sigm(this.modelResult[i * output_size + 1]);
if (curConf > 0.55 && curConf > maxConf) {
maxConf = curConf;
maxIndex = i;
}
}
this.maxIndex = maxIndex;
function sigm(value) {
return 1.0 / (1.0 + Math.exp(0.0 - value));
}
}
decodeAnchor() {
const index = this.maxIndex;
const anchors = this.anchors;
this.pxmin = anchors[index][0];
this.pymin = anchors[index][1];
this.pxmax = anchors[index][2];
this.pymax = anchors[index][3];
}
decodeKp() {
const modelResult = this.modelResult;
const index = this.maxIndex;
const px = (this.pxmin + this.pxmax) / 2;
const py = (this.pymin + this.pymax) / 2;
const pw = this.pxmax - this.pxmin;
const ph = this.pymax - this.pymin;
const prior_var = 0.1;
const kp: number[][] = [[], [], []];
kp[0][0] = (modelResult[index * this.output_size + 6] * pw * prior_var + px) * 256;
kp[0][1] = (modelResult[index * this.output_size + 8] * ph * prior_var + py) * 256;
kp[2][0] = (modelResult[index * this.output_size + 7] * pw * prior_var + px) * 256;
kp[2][1] = (modelResult[index * this.output_size + 9] * ph * prior_var + py) * 256;
this.kp = kp;
}
decodeBox() {
const modelResult = this.modelResult;
const output_size = this.output_size || 10;
const pw = this.pxmax - this.pxmin;
const ph = this.pymax - this.pymin;
const px = this.pxmin + pw / 2;
const py = this.pymin + ph / 2;
const prior_var = 0.1;
const index = this.maxIndex;
const ox = modelResult[output_size * index + 2];
const oy = modelResult[output_size * index + 3];
const ow = modelResult[output_size * index + 4];
const oh = modelResult[output_size * index + 5];
const tx = ox * prior_var * pw + px;
const ty = oy * prior_var * ph + py;
const tw = this.tw = Math.pow(2.71828182, prior_var * ow) * pw;
const th = this.th = Math.pow(2.71828182, prior_var * oh) * ph;
const box: number[][] = [[], [], [], []];
box[0][0] = (tx - tw / 2) * 256;
box[0][1] = (ty - th / 2) * 256;
box[1][0] = (tx + tw / 2) * 256;
box[1][1] = (ty - th / 2) * 256;
box[2][0] = (tx + tw / 2) * 256;
box[2][1] = (ty + th / 2) * 256;
box[3][0] = (tx - tw / 2) * 256;
box[3][1] = (ty + th / 2) * 256;
this.box = box;
}
decodeTriangle() {
const box_enlarge = 1.04;
const side = Math.max(this.tw * 256, this.th * 256) * (box_enlarge);
const dir_v: number[] = [];
const kp = this.kp;
const triangle: number[][] = [[], [], []];
const dir_v_r: number[] = [];
dir_v[0] = kp[2][0] - kp[0][0];
dir_v[1] = kp[2][1] - kp[0][1];
const sq = Math.sqrt(Math.pow(dir_v[0], 2) + Math.pow(dir_v[1], 2)) || 1;
dir_v[0] = dir_v[0] / sq;
dir_v[1] = dir_v[1] / sq;
dir_v_r[0] = dir_v[0] * 0 + dir_v[1] * 1;
dir_v_r[1] = dir_v[0] * -1 + dir_v[1] * 0;
triangle[0][0] = kp[2][0];
triangle[0][1] = kp[2][1];
triangle[1][0] = kp[2][0] + dir_v[0] * side;
triangle[1][1] = kp[2][1] + dir_v[1] * side;
triangle[2][0] = kp[2][0] + dir_v_r[0] * side;
triangle[2][1] = kp[2][1] + dir_v_r[1] * side;
this.triangle = triangle;
}
decodeSource() {
const kp = this.kp;
const box_shift = 0.0;
const tmp0 = (kp[0][0] - kp[2][0]) * box_shift;
const tmp1 = (kp[0][1] - kp[2][1]) * box_shift;
const source: number[][] = [[], [], []];
for (let i = 0; i < 3; i++) {
source[i][0] = this.triangle[i][0] - tmp0;
source[i][1] = this.triangle[i][1] - tmp1;
}
this.source = source;
}
outputResult() {
for (let i = 0; i < 4; i++) {
this.detectResult[i][0] = this.box[i][0];
this.detectResult[i][1] = this.box[i][1];
}
this.detectResult[4][0] = this.kp[0][0];
this.detectResult[4][1] = this.kp[0][1];
this.detectResult[6][0] = this.kp[2][0];
this.detectResult[6][1] = this.kp[2][1];
for (let i = 0; i < 3; i++) {
this.detectResult[i + 11][0] = this.source[i][0];
this.detectResult[i + 11][1] = this.source[i][1];
}
}
getaffinetransform() {
// 图像上的原始坐标点。需要对所有坐标进行归一化_x = (x - 128) / 128, _y = (128 - y) / 128
// 坐标矩阵
// x1 x2 x3
// y1 y2 y3
// z1 z2 z3
const originPoints = [].concat(this.detectResult[11][0] as number / 128 - 1 as never)
.concat(this.detectResult[12][0] / 128 - 1 as number as never)
.concat(this.detectResult[13][0] / 128 - 1 as number as never)
.concat(1 - this.detectResult[11][1] / 128 as number as never)
.concat(1 - this.detectResult[12][1] / 128 as number as never)
.concat(1 - this.detectResult[13][1] / 128 as number as never)
.concat([1, 1, 1] as never[]);
// originPoints = [0, 0, -1, .1, 1.1, 0.1, 1, 1, 1];
const matrixA = new Matrix(3, 3, originPoints);
// 转化后的点[128, 128, 0, 128, 0, 128] [0, 0, -1, 0, 1, 0]
const matrixB = new Matrix(2, 3, [0, 0, -1, 0, -1, 0]);
// M * A = B => M = B * A逆
let _matrixA: any = inverseMatrix(matrixA.data);
_matrixA = new Matrix(3, 3, _matrixA);
this.mtr = Matrix_Product(matrixB, _matrixA);
}
async warpAffine() {
const ctx = this.originCanvas.getContext('2d');
this.originImageData = ctx!.getImageData(0, 0, 256, 256);
const imageDataArr = await WarpAffine.main({
imageData: this.originImageData,
mtr: this.mtr.data,
input: {
width: 256,
height: 256
},
output: {
width: 224,
height: 224
}
});
this.imageData = new ImageData(Uint8ClampedArray.from(imageDataArr), 224, 224);
}
allReshapeToRGB(paddle) {
const data = paddle.mediaProcessor.allReshapeToRGB(this.imageData, {
gapFillWith: '#000',
mean: [0, 0, 0],
scale: 224,
std: [1, 1, 1],
targetShape: [1, 3, 224, 224],
normalizeType: 1
});
this.feed = [{
data: new Float32Array(data),
shape: [1, 3, 224, 224],
name: 'image',
canvas: this.originImageData
}];
}
}
class Matrix {
private row: number;
private col: number;
data: number[] | number[][];
constructor(row, col, arr: number[] | number[][] = []) {
// 行
this.row = row;
// 列
this.col = col;
if (arr[0] && arr[0] instanceof Array) {
this.data = arr;
}
else {
this.data = [];
const _arr: number[] = [].concat(arr as never[]);
// 创建row个元素的空数组
const Matrix = new Array(row);
// 对第一层数组遍历
for (let i = 0; i < row; i++) {
// 每一行创建col列的空数组
Matrix[i] = new Array(col).fill('');
Matrix[i].forEach((_item, index, cur) => {
cur[index] = _arr.shift() || 0;
});
}
// 将矩阵保存到this.data上
this.data = Matrix;
}
}
}
const Matrix_Product = (A, B) => {
const tempMatrix = new Matrix(A.row, B.col);
if (A.col === B.row) {
for (let i = 0; i < A.row; i++) {
for (let j = 0; j < B.col; j++) {
tempMatrix.data[i][j] = 0;
for (let n = 0; n < A.col; n++) {
tempMatrix.data[i][j] += A.data[i][n] * B.data[n][j];
}
tempMatrix.data[i][j] = tempMatrix.data[i][j].toFixed(5);
}
}
return tempMatrix;
}
return false;
};
// 求行列式
const determinant = matrix => {
const order = matrix.length;
let cofactor;
let result = 0;
if (order === 1) {
return matrix[0][0];
}
for (let i = 0; i < order; i++) {
cofactor = [];
for (let j = 0; j < order - 1; j++) {
cofactor[j] = [];
for (let k = 0; k < order - 1; k++) {
cofactor[j][k] = matrix[j + 1][k < i ? k : k + 1];
}
}
result += matrix[0][i] * Math.pow(-1, i) * determinant(cofactor);
}
return result;
};
// 矩阵数乘
function scalarMultiply(num, matrix) {
const row = matrix.length;
const col = matrix[0].length;
const result: number[][] = [];
for (let i = 0; i < row; i++) {
result[i] = [];
for (let j = 0; j < col; j++) {
result[i][j] = num * matrix[i][j];
}
}
return result;
}
// 求逆矩阵
function inverseMatrix(matrix) {
if (determinant(matrix) === 0) {
return false;
}
// 求代数余子式
function cofactor(matrix, row, col) {
const order = matrix.length;
const new_matrix: number[][] = [];
let _row;
let _col;
for (let i = 0; i < order - 1; i++) {
new_matrix[i] = [];
_row = i < row ? i : i + 1;
for (let j = 0; j < order - 1; j++) {
_col = j < col ? j : j + 1;
new_matrix[i][j] = matrix[_row][_col];
}
}
return Math.pow(-1, row + col) * determinant(new_matrix);
}
const order = matrix.length;
const adjoint: number[][] = [];
for (let i = 0; i < order; i++) {
adjoint[i] = [];
for (let j = 0; j < order; j++) {
adjoint[i][j] = cofactor(matrix, j, i);
}
}
return scalarMultiply(1 / determinant(matrix), adjoint);
}
export default DetectProcess;

View File

@@ -0,0 +1 @@
declare module '*.txt';

View File

@@ -0,0 +1,73 @@
/**
* @file gesture model
*/
import { Runner } from '@paddlejs/paddlejs-core';
import '@paddlejs/paddlejs-backend-webgl';
import WarpAffine from './warpAffine';
import DetectProcess from './detectProcess';
import LMProcess from './LMProcess';
import anchor from './anchor_small.txt';
let box = null;
let detectRunner = null as Runner;
let recRunner = null as Runner;
let anchorResults = null;
const detFeedShape = 256;
const canvas = document.createElement('canvas') as HTMLCanvasElement;
initCanvas();
function initCanvas() {
canvas.width = detFeedShape;
canvas.height = detFeedShape;
canvas.style.position = 'absolute';
canvas.style.top = '0';
canvas.style.left = '0';
canvas.style.zIndex = '-1';
canvas.style.opacity = '0';
document.body.appendChild(canvas);
}
export async function load() {
anchorResults = anchor.replace(/\s+/g, ',').split(',').map(item => +item);
detectRunner = new Runner({
modelPath: 'https://paddlejs.bj.bcebos.com/models/fuse/gesture/gesture_det_fuse_activation/model.json',
webglFeedProcess: true
});
const detectInit = detectRunner.init();
recRunner = new Runner({
modelPath: 'https://paddlejs.bj.bcebos.com/models/fuse/gesture/gesture_rec_fuse_activation/model.json'
});
const recInit = recRunner.init();
WarpAffine.init({
width: 224,
height: 224
});
return await Promise.all([detectInit, recInit]);
}
export async function classify(image): Promise<{ box: string | number[][], type: string }> {
canvas.getContext('2d')!.drawImage(image, 0, 0, detFeedShape, detFeedShape);
const res = await detectRunner.predict(image);
const post = new DetectProcess(res, canvas);
const result = {
box: '',
type: ''
};
box = await post.outputBox(anchorResults);
if (box) {
// 手势框选位置
result.box = box;
const feed = await post.outputFeed(recRunner);
const res2 = await recRunner.predictWithFeed(feed);
const lmProcess = new LMProcess(res2);
lmProcess.output();
// 识别结果
result.type = lmProcess.type || '';
}
return result;
}

View File

@@ -0,0 +1,156 @@
const VSHADER_SOURCE = `attribute vec4 a_Position;
attribute vec2 a_TexCoord;
uniform mat4 translation;
varying vec2 v_TexCoord;
void main() {
gl_Position = translation * a_Position;
v_TexCoord = a_TexCoord;
}`;
const FSHADER_SOURCE = `precision mediump float;
uniform sampler2D u_Sampler;
varying vec2 v_TexCoord;
void main() {
gl_FragColor = texture2D(u_Sampler, v_TexCoord);
}`;
let imgData = null;
let mtrData: number[][] = null as unknown as number[][];
let gl: any = null;
function init(output) {
const canvas = document.createElement('canvas');
canvas.width = output.width;
canvas.height = output.height;
gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
// 初始化之前先加载图片
if (!initShaders(gl, VSHADER_SOURCE, FSHADER_SOURCE)) {
throw new Error('initShaders false');
}
}
function main(opt) {
const {
imageData,
mtr,
output
} = opt;
mtrData = mtr;
imgData = imageData;
gl.clearColor(0, 0, 0, 1);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
const n = initVertexBuffers(gl);
initTextures(gl, n, 0);
const outputData = new Uint8Array(output.width * output.height * 4);
gl.readPixels(0, 0, output.width, output.height, gl.RGBA, gl.UNSIGNED_BYTE, outputData);
return outputData;
}
function initVertexBuffers(gl) {
// 顶点坐标和纹理图像坐标
const vertices = new Float32Array([
// webgl坐标纹理坐标
-1.0, 1.0, 0.0, 0.0, 1.0,
-1.0, -1.0, 0.0, 0.0, 0.0,
1.0, 1.0, 0.0, 1.0, 1.0,
1.0, -1.0, 0.0, 1.0, 0.0
]);
const FSIZE = vertices.BYTES_PER_ELEMENT;
const vertexBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW);
const aPosition = gl.getAttribLocation(gl.program, 'a_Position');
const aTexCoord = gl.getAttribLocation(gl.program, 'a_TexCoord');
const xformMatrix = new Float32Array([
mtrData[0][0], mtrData[1][0], 0.0, 0.0,
mtrData[0][1], mtrData[1][1], 0.0, 0.0,
0.0, 0.0, 0.0, 1.0,
mtrData[0][2], mtrData[1][2], 0.0, 1.0
]);
const translation = gl.getUniformLocation(gl.program, 'translation');
gl.uniformMatrix4fv(translation, false, xformMatrix);
gl.vertexAttribPointer(aPosition, 3, gl.FLOAT, false, FSIZE * 5, 0);
gl.enableVertexAttribArray(aPosition);
gl.vertexAttribPointer(aTexCoord, 2, gl.FLOAT, false, FSIZE * 5, FSIZE * 3);
gl.enableVertexAttribArray(aTexCoord);
return 4;
}
function initTextures(gl, n, index) {
// 创建纹理对象
const texture = gl.createTexture();
const uSampler = gl.getUniformLocation(gl.program, 'u_Sampler');
// WebGL纹理坐标中的纵轴方向和PNGJPG等图片容器的Y轴方向是反的所以先反转Y轴
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1);
// 激活纹理单元开启index号纹理单元
gl.activeTexture(gl[`TEXTURE${index}`]);
// 绑定纹理对象
gl.bindTexture(gl.TEXTURE_2D, texture);
// 配置纹理对象的参数
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
// 将纹理图像分配给纹理对象
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, imgData);
// 将纹理单元编号传给着色器
gl.uniform1i(uSampler, index);
// 绘制
gl.drawArrays(gl.TRIANGLE_STRIP, 0, n);
}
function initShaders(gl, vshader, fshader) {
const vertexShader = createShader(gl, vshader, gl.VERTEX_SHADER);
const fragmentShader = createShader(gl, fshader, gl.FRAGMENT_SHADER);
const program = gl.createProgram();
gl.attachShader(program, vertexShader);
gl.attachShader(program, fragmentShader);
if (!program) {
console.log('无法创建程序对象');
return false;
}
gl.linkProgram(program);
gl.useProgram(program);
gl.program = program;
return true;
}
function createShader(gl, sourceCode, type) {
// Compiles either a shader of type gl.VERTEX_SHADER or gl.FRAGMENT_SHADER
const shader = gl.createShader(type);
gl.shaderSource(shader, sourceCode);
gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
const info = gl.getShaderInfoLog(shader);
throw Error('Could not compile WebGL program. \n\n' + info);
}
return shader;
}
export default {
main,
init
};

View File

@@ -0,0 +1,12 @@
import assert from 'assert'
describe('Example:', () => {
/**
* Example
*/
describe('ExampleTest', () => {
test('Hello World!', () => {
assert.strictEqual('Hello World!', 'Hello World!')
})
})
})

View File

@@ -0,0 +1,11 @@
{
"compilerOptions": {
"baseUrl": "./",
"resolveJsonModule": true
},
"include": [
"**/*.ts",
"**/*.js",
".eslintrc.js"
]
}

View File

@@ -0,0 +1,31 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig to read more about this file */
"target": "ESNext", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
"lib": [
"ESNext",
"DOM"
], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
"moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */
"baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
"resolveJsonModule": true, /* Enable importing .json files. */
"allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
"declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
"declarationMap": true, /* Create sourcemaps for d.ts files. */
"sourceMap": true, /* Create source map files for emitted JavaScript files. */
"outDir": "./lib", /* Specify an output folder for all emitted files. */
"removeComments": false, /* Disable emitting comments. */
"allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
"esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
"forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */
"strict": true, /* Enable all strict type-checking options. */
"noImplicitAny": false, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
"skipLibCheck": true /* Skip type checking all .d.ts files. */
},
"include": [
"src"
]
}