mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-10-06 17:17:14 +08:00
[Other] Refactor js submodule (#415)
* Refactor js submodule * Remove change-log * Update ocr module * Update ocr-detection module * Update ocr-detection module * Remove change-log
This commit is contained in:
chenqianhe
@@ -0,0 +1,100 @@
|
||||
export default class LMProcess {
|
||||
private input_n: number;
|
||||
private input_h: number;
|
||||
private input_w: number;
|
||||
private input_c: number;
|
||||
private class_dim: number;
|
||||
private points_num: number;
|
||||
private result: any;
|
||||
private conf!: number;
|
||||
private kp: any;
|
||||
private forefinger: any;
|
||||
private output_softmax: any;
|
||||
type!: string;
|
||||
|
||||
constructor(result) {
|
||||
/*
|
||||
* result[0] :是否为手
|
||||
* result[1:43]: 手的21个关键点
|
||||
* result[43:49]: 几中手势分类，包含但不限于石头剪刀布，为了提升准确度
|
||||
* this.kp decode result得出的21个手指关键点，this.kp[8]为食指
|
||||
* this.conf 是否为手，越大，越可能是手
|
||||
*/
|
||||
this.input_n = 1;
|
||||
this.input_h = 224;
|
||||
this.input_w = 224;
|
||||
this.input_c = 3;
|
||||
this.class_dim = 6;
|
||||
this.points_num = 1;
|
||||
this.result = result;
|
||||
}
|
||||
|
||||
sigm(value) {
|
||||
return 1.0 / (1.0 + Math.exp(0.0 - value));
|
||||
}
|
||||
|
||||
decodeConf() {
|
||||
this.conf = this.sigm(this.result[0]);
|
||||
}
|
||||
|
||||
decodeKp() {
|
||||
// 21个关键点，下标1开始
|
||||
const offset = 1;
|
||||
const result = this.result;
|
||||
this.kp = [];
|
||||
for (let i = 0; i < this.points_num; i++) {
|
||||
const arr: number[] = [];
|
||||
arr.push((result[offset + i * 2] + 0.5) * this.input_h);
|
||||
arr.push((result[offset + i * 2 + 1] + 0.5) * this.input_h);
|
||||
this.kp.push(arr);
|
||||
}
|
||||
this.forefinger = this.kp[0];
|
||||
}
|
||||
|
||||
softMax() {
|
||||
let max = 0;
|
||||
let sum = 0;
|
||||
const offset = 2;
|
||||
const class_dim = this.class_dim = 7;
|
||||
const result = this.result;
|
||||
const output_softmax = new Array(7).fill(null);
|
||||
|
||||
for (let i = 0; i < class_dim; i++) {
|
||||
if (max < result[i + offset]) {
|
||||
max = result[i + offset];
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < class_dim; i++) {
|
||||
output_softmax[i] = Math.exp(result[i + offset] - max);
|
||||
sum += output_softmax[i];
|
||||
}
|
||||
|
||||
for (let i = 0; i < class_dim; i++) {
|
||||
output_softmax[i] /= sum;
|
||||
}
|
||||
|
||||
this.output_softmax = output_softmax;
|
||||
}
|
||||
|
||||
output() {
|
||||
this.decodeKp();
|
||||
this.softMax();
|
||||
|
||||
let label_index = 0;
|
||||
let max_pro = this.output_softmax[0];
|
||||
for (let i = 1; i < this.class_dim; i++) {
|
||||
if (max_pro < this.output_softmax[i]) {
|
||||
label_index = i;
|
||||
max_pro = this.output_softmax[i];
|
||||
}
|
||||
}
|
||||
// 最后一位：有无手
|
||||
if (label_index !== 0 && label_index !== this.class_dim - 1 && max_pro > 0.9) {
|
||||
const ges = ['其他', '布', '剪刀', '石头', '1', 'ok'];
|
||||
this.type = ges[label_index];
|
||||
return;
|
||||
}
|
||||
this.type = 'other';
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,414 @@
|
||||
import WarpAffine from './warpAffine';
|
||||
|
||||
class DetectProcess {
|
||||
private modelResult: any;
|
||||
private output_size: number;
|
||||
private anchor_num: number;
|
||||
private detectResult: any;
|
||||
private hasHand: number;
|
||||
private originCanvas: HTMLCanvasElement;
|
||||
private originImageData: ImageData;
|
||||
private maxIndex!: number;
|
||||
private box: any;
|
||||
private feed: any;
|
||||
private anchors: any;
|
||||
private mtr: any;
|
||||
private source!: any[][];
|
||||
private imageData!: ImageData;
|
||||
private pxmin: any;
|
||||
private pxmax: any;
|
||||
private pymin: any;
|
||||
private pymax: any;
|
||||
private kp!: number[][];
|
||||
private tw!: number;
|
||||
private th!: number;
|
||||
private triangle: any;
|
||||
private results: any;
|
||||
|
||||
constructor(result, canvas) {
|
||||
this.modelResult = result;
|
||||
this.output_size = 10;
|
||||
this.anchor_num = 1920;
|
||||
this.detectResult = new Array(14).fill('').map(() => []);
|
||||
this.hasHand = 0;
|
||||
this.originCanvas = canvas;
|
||||
const ctx = this.originCanvas.getContext('2d');
|
||||
this.originImageData = ctx!.getImageData(0, 0, 256, 256);
|
||||
}
|
||||
|
||||
async outputBox(results) {
|
||||
this.getMaxIndex();
|
||||
if (this.maxIndex === -1) {
|
||||
this.hasHand = 0;
|
||||
return false;
|
||||
}
|
||||
this.hasHand = 1;
|
||||
await this.splitAnchors(results);
|
||||
this.decodeAnchor();
|
||||
// 求关键点
|
||||
this.decodeKp();
|
||||
// 求手势框
|
||||
this.decodeBox();
|
||||
return this.box;
|
||||
}
|
||||
|
||||
async outputFeed(paddle) {
|
||||
this.decodeTriangle();
|
||||
this.decodeSource();
|
||||
this.outputResult();
|
||||
// 提取仿射变化矩阵
|
||||
this.getaffinetransform();
|
||||
// 对图片进行仿射变换
|
||||
await this.warpAffine();
|
||||
this.allReshapeToRGB(paddle);
|
||||
return this.feed;
|
||||
}
|
||||
|
||||
async splitAnchors(results) {
|
||||
this.results = results;
|
||||
const anchors: number[][] = new Array(this.anchor_num).fill('').map(() => []);
|
||||
const anchor_num = this.anchor_num;
|
||||
for (let i = 0; i < anchor_num; i++) {
|
||||
const tmp0 = results[i * 4];
|
||||
const tmp1 = results[i * 4 + 1];
|
||||
const tmp2 = results[i * 4 + 2];
|
||||
const tmp3 = results[i * 4 + 3];
|
||||
anchors[i] = [
|
||||
tmp0 - tmp2 / 2.0,
|
||||
tmp1 - tmp3 / 2.0,
|
||||
tmp0 + tmp2 / 2.0,
|
||||
tmp1 + tmp3 / 2.0
|
||||
];
|
||||
if (anchors[i][0] < 0) {
|
||||
anchors[i][0] = 0;
|
||||
}
|
||||
if (anchors[i][0] > 1) {
|
||||
anchors[i][0] = 1;
|
||||
}
|
||||
if (anchors[i][1] < 0) {
|
||||
anchors[i][1] = 0;
|
||||
}
|
||||
if (anchors[i][1] > 1) {
|
||||
anchors[i][1] = 1;
|
||||
}
|
||||
if (anchors[i][2] < 0) {
|
||||
anchors[i][2] = 0;
|
||||
}
|
||||
if (anchors[i][2] > 1) {
|
||||
anchors[i][2] = 1;
|
||||
}
|
||||
if (anchors[i][3] < 0) {
|
||||
anchors[i][3] = 0;
|
||||
}
|
||||
if (anchors[i][3] > 1) {
|
||||
anchors[i][3] = 1;
|
||||
}
|
||||
}
|
||||
this.anchors = anchors;
|
||||
}
|
||||
|
||||
getMaxIndex() {
|
||||
let maxIndex = -1;
|
||||
let maxConf = -1;
|
||||
let curConf = -2.0;
|
||||
const output_size = 10;
|
||||
|
||||
for (let i = 0; i < this.anchor_num; i++) {
|
||||
curConf = sigm(this.modelResult[i * output_size + 1]);
|
||||
if (curConf > 0.55 && curConf > maxConf) {
|
||||
maxConf = curConf;
|
||||
maxIndex = i;
|
||||
}
|
||||
}
|
||||
this.maxIndex = maxIndex;
|
||||
function sigm(value) {
|
||||
return 1.0 / (1.0 + Math.exp(0.0 - value));
|
||||
}
|
||||
}
|
||||
|
||||
decodeAnchor() {
|
||||
const index = this.maxIndex;
|
||||
const anchors = this.anchors;
|
||||
this.pxmin = anchors[index][0];
|
||||
this.pymin = anchors[index][1];
|
||||
this.pxmax = anchors[index][2];
|
||||
this.pymax = anchors[index][3];
|
||||
}
|
||||
|
||||
decodeKp() {
|
||||
const modelResult = this.modelResult;
|
||||
const index = this.maxIndex;
|
||||
const px = (this.pxmin + this.pxmax) / 2;
|
||||
const py = (this.pymin + this.pymax) / 2;
|
||||
const pw = this.pxmax - this.pxmin;
|
||||
const ph = this.pymax - this.pymin;
|
||||
const prior_var = 0.1;
|
||||
const kp: number[][] = [[], [], []];
|
||||
kp[0][0] = (modelResult[index * this.output_size + 6] * pw * prior_var + px) * 256;
|
||||
kp[0][1] = (modelResult[index * this.output_size + 8] * ph * prior_var + py) * 256;
|
||||
kp[2][0] = (modelResult[index * this.output_size + 7] * pw * prior_var + px) * 256;
|
||||
kp[2][1] = (modelResult[index * this.output_size + 9] * ph * prior_var + py) * 256;
|
||||
this.kp = kp;
|
||||
}
|
||||
|
||||
decodeBox() {
|
||||
const modelResult = this.modelResult;
|
||||
const output_size = this.output_size || 10;
|
||||
const pw = this.pxmax - this.pxmin;
|
||||
const ph = this.pymax - this.pymin;
|
||||
const px = this.pxmin + pw / 2;
|
||||
const py = this.pymin + ph / 2;
|
||||
const prior_var = 0.1;
|
||||
const index = this.maxIndex;
|
||||
|
||||
const ox = modelResult[output_size * index + 2];
|
||||
const oy = modelResult[output_size * index + 3];
|
||||
const ow = modelResult[output_size * index + 4];
|
||||
const oh = modelResult[output_size * index + 5];
|
||||
|
||||
const tx = ox * prior_var * pw + px;
|
||||
const ty = oy * prior_var * ph + py;
|
||||
const tw = this.tw = Math.pow(2.71828182, prior_var * ow) * pw;
|
||||
const th = this.th = Math.pow(2.71828182, prior_var * oh) * ph;
|
||||
const box: number[][] = [[], [], [], []];
|
||||
box[0][0] = (tx - tw / 2) * 256;
|
||||
box[0][1] = (ty - th / 2) * 256;
|
||||
box[1][0] = (tx + tw / 2) * 256;
|
||||
box[1][1] = (ty - th / 2) * 256;
|
||||
box[2][0] = (tx + tw / 2) * 256;
|
||||
box[2][1] = (ty + th / 2) * 256;
|
||||
box[3][0] = (tx - tw / 2) * 256;
|
||||
box[3][1] = (ty + th / 2) * 256;
|
||||
this.box = box;
|
||||
}
|
||||
|
||||
decodeTriangle() {
|
||||
const box_enlarge = 1.04;
|
||||
const side = Math.max(this.tw * 256, this.th * 256) * (box_enlarge);
|
||||
const dir_v: number[] = [];
|
||||
const kp = this.kp;
|
||||
const triangle: number[][] = [[], [], []];
|
||||
const dir_v_r: number[] = [];
|
||||
|
||||
dir_v[0] = kp[2][0] - kp[0][0];
|
||||
dir_v[1] = kp[2][1] - kp[0][1];
|
||||
const sq = Math.sqrt(Math.pow(dir_v[0], 2) + Math.pow(dir_v[1], 2)) || 1;
|
||||
dir_v[0] = dir_v[0] / sq;
|
||||
dir_v[1] = dir_v[1] / sq;
|
||||
|
||||
dir_v_r[0] = dir_v[0] * 0 + dir_v[1] * 1;
|
||||
dir_v_r[1] = dir_v[0] * -1 + dir_v[1] * 0;
|
||||
triangle[0][0] = kp[2][0];
|
||||
triangle[0][1] = kp[2][1];
|
||||
triangle[1][0] = kp[2][0] + dir_v[0] * side;
|
||||
triangle[1][1] = kp[2][1] + dir_v[1] * side;
|
||||
triangle[2][0] = kp[2][0] + dir_v_r[0] * side;
|
||||
triangle[2][1] = kp[2][1] + dir_v_r[1] * side;
|
||||
this.triangle = triangle;
|
||||
}
|
||||
|
||||
decodeSource() {
|
||||
const kp = this.kp;
|
||||
const box_shift = 0.0;
|
||||
const tmp0 = (kp[0][0] - kp[2][0]) * box_shift;
|
||||
const tmp1 = (kp[0][1] - kp[2][1]) * box_shift;
|
||||
const source: number[][] = [[], [], []];
|
||||
for (let i = 0; i < 3; i++) {
|
||||
source[i][0] = this.triangle[i][0] - tmp0;
|
||||
source[i][1] = this.triangle[i][1] - tmp1;
|
||||
}
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
outputResult() {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
this.detectResult[i][0] = this.box[i][0];
|
||||
this.detectResult[i][1] = this.box[i][1];
|
||||
}
|
||||
this.detectResult[4][0] = this.kp[0][0];
|
||||
this.detectResult[4][1] = this.kp[0][1];
|
||||
this.detectResult[6][0] = this.kp[2][0];
|
||||
this.detectResult[6][1] = this.kp[2][1];
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
this.detectResult[i + 11][0] = this.source[i][0];
|
||||
this.detectResult[i + 11][1] = this.source[i][1];
|
||||
}
|
||||
}
|
||||
|
||||
getaffinetransform() {
|
||||
// 图像上的原始坐标点。需要对所有坐标进行归一化，_x = (x - 128) / 128, _y = (128 - y) / 128
|
||||
// 坐标矩阵
|
||||
// x1 x2 x3
|
||||
// y1 y2 y3
|
||||
// z1 z2 z3
|
||||
const originPoints = [].concat(this.detectResult[11][0] as number / 128 - 1 as never)
|
||||
.concat(this.detectResult[12][0] / 128 - 1 as number as never)
|
||||
.concat(this.detectResult[13][0] / 128 - 1 as number as never)
|
||||
.concat(1 - this.detectResult[11][1] / 128 as number as never)
|
||||
.concat(1 - this.detectResult[12][1] / 128 as number as never)
|
||||
.concat(1 - this.detectResult[13][1] / 128 as number as never)
|
||||
.concat([1, 1, 1] as never[]);
|
||||
// originPoints = [0, 0, -1, .1, 1.1, 0.1, 1, 1, 1];
|
||||
const matrixA = new Matrix(3, 3, originPoints);
|
||||
// 转化后的点[128, 128, 0, 128, 0, 128] [0, 0, -1, 0, 1, 0]
|
||||
const matrixB = new Matrix(2, 3, [0, 0, -1, 0, -1, 0]);
|
||||
// M * A = B => M = B * A逆
|
||||
let _matrixA: any = inverseMatrix(matrixA.data);
|
||||
_matrixA = new Matrix(3, 3, _matrixA);
|
||||
this.mtr = Matrix_Product(matrixB, _matrixA);
|
||||
}
|
||||
|
||||
async warpAffine() {
|
||||
const ctx = this.originCanvas.getContext('2d');
|
||||
this.originImageData = ctx!.getImageData(0, 0, 256, 256);
|
||||
const imageDataArr = await WarpAffine.main({
|
||||
imageData: this.originImageData,
|
||||
mtr: this.mtr.data,
|
||||
input: {
|
||||
width: 256,
|
||||
height: 256
|
||||
},
|
||||
output: {
|
||||
width: 224,
|
||||
height: 224
|
||||
}
|
||||
});
|
||||
this.imageData = new ImageData(Uint8ClampedArray.from(imageDataArr), 224, 224);
|
||||
}
|
||||
|
||||
allReshapeToRGB(paddle) {
|
||||
const data = paddle.mediaProcessor.allReshapeToRGB(this.imageData, {
|
||||
gapFillWith: '#000',
|
||||
mean: [0, 0, 0],
|
||||
scale: 224,
|
||||
std: [1, 1, 1],
|
||||
targetShape: [1, 3, 224, 224],
|
||||
normalizeType: 1
|
||||
});
|
||||
this.feed = [{
|
||||
data: new Float32Array(data),
|
||||
shape: [1, 3, 224, 224],
|
||||
name: 'image',
|
||||
canvas: this.originImageData
|
||||
}];
|
||||
}
|
||||
}
|
||||
|
||||
class Matrix {
|
||||
private row: number;
|
||||
private col: number;
|
||||
data: number[] | number[][];
|
||||
|
||||
constructor(row, col, arr: number[] | number[][] = []) {
|
||||
// 行
|
||||
this.row = row;
|
||||
// 列
|
||||
this.col = col;
|
||||
if (arr[0] && arr[0] instanceof Array) {
|
||||
this.data = arr;
|
||||
}
|
||||
else {
|
||||
this.data = [];
|
||||
const _arr: number[] = [].concat(arr as never[]);
|
||||
// 创建row个元素的空数组
|
||||
const Matrix = new Array(row);
|
||||
// 对第一层数组遍历
|
||||
for (let i = 0; i < row; i++) {
|
||||
// 每一行创建col列的空数组
|
||||
Matrix[i] = new Array(col).fill('');
|
||||
Matrix[i].forEach((_item, index, cur) => {
|
||||
cur[index] = _arr.shift() || 0;
|
||||
});
|
||||
}
|
||||
// 将矩阵保存到this.data上
|
||||
this.data = Matrix;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const Matrix_Product = (A, B) => {
|
||||
const tempMatrix = new Matrix(A.row, B.col);
|
||||
if (A.col === B.row) {
|
||||
for (let i = 0; i < A.row; i++) {
|
||||
for (let j = 0; j < B.col; j++) {
|
||||
tempMatrix.data[i][j] = 0;
|
||||
for (let n = 0; n < A.col; n++) {
|
||||
tempMatrix.data[i][j] += A.data[i][n] * B.data[n][j];
|
||||
}
|
||||
tempMatrix.data[i][j] = tempMatrix.data[i][j].toFixed(5);
|
||||
}
|
||||
}
|
||||
return tempMatrix;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// 求行列式
|
||||
const determinant = matrix => {
|
||||
const order = matrix.length;
|
||||
let cofactor;
|
||||
let result = 0;
|
||||
if (order === 1) {
|
||||
return matrix[0][0];
|
||||
}
|
||||
for (let i = 0; i < order; i++) {
|
||||
cofactor = [];
|
||||
for (let j = 0; j < order - 1; j++) {
|
||||
cofactor[j] = [];
|
||||
for (let k = 0; k < order - 1; k++) {
|
||||
cofactor[j][k] = matrix[j + 1][k < i ? k : k + 1];
|
||||
}
|
||||
}
|
||||
result += matrix[0][i] * Math.pow(-1, i) * determinant(cofactor);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
// 矩阵数乘
|
||||
function scalarMultiply(num, matrix) {
|
||||
const row = matrix.length;
|
||||
const col = matrix[0].length;
|
||||
const result: number[][] = [];
|
||||
for (let i = 0; i < row; i++) {
|
||||
result[i] = [];
|
||||
for (let j = 0; j < col; j++) {
|
||||
result[i][j] = num * matrix[i][j];
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// 求逆矩阵
|
||||
function inverseMatrix(matrix) {
|
||||
if (determinant(matrix) === 0) {
|
||||
return false;
|
||||
}
|
||||
// 求代数余子式
|
||||
function cofactor(matrix, row, col) {
|
||||
const order = matrix.length;
|
||||
const new_matrix: number[][] = [];
|
||||
let _row;
|
||||
let _col;
|
||||
for (let i = 0; i < order - 1; i++) {
|
||||
new_matrix[i] = [];
|
||||
_row = i < row ? i : i + 1;
|
||||
for (let j = 0; j < order - 1; j++) {
|
||||
_col = j < col ? j : j + 1;
|
||||
new_matrix[i][j] = matrix[_row][_col];
|
||||
}
|
||||
}
|
||||
return Math.pow(-1, row + col) * determinant(new_matrix);
|
||||
}
|
||||
const order = matrix.length;
|
||||
const adjoint: number[][] = [];
|
||||
for (let i = 0; i < order; i++) {
|
||||
adjoint[i] = [];
|
||||
for (let j = 0; j < order; j++) {
|
||||
adjoint[i][j] = cofactor(matrix, j, i);
|
||||
}
|
||||
}
|
||||
return scalarMultiply(1 / determinant(matrix), adjoint);
|
||||
}
|
||||
|
||||
export default DetectProcess;
|
||||
1
examples/application/js/package/packages/paddlejs-models/gesture/src/global.d.ts
vendored
Normal file
1
examples/application/js/package/packages/paddlejs-models/gesture/src/global.d.ts
vendored
Normal file
@@ -0,0 +1 @@
|
||||
declare module '*.txt';
|
||||
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* @file gesture model
|
||||
*/
|
||||
|
||||
import { Runner } from '@paddlejs/paddlejs-core';
|
||||
import '@paddlejs/paddlejs-backend-webgl';
|
||||
import WarpAffine from './warpAffine';
|
||||
import DetectProcess from './detectProcess';
|
||||
import LMProcess from './LMProcess';
|
||||
import anchor from './anchor_small.txt';
|
||||
|
||||
let box = null;
|
||||
let detectRunner = null as Runner;
|
||||
let recRunner = null as Runner;
|
||||
let anchorResults = null;
|
||||
const detFeedShape = 256;
|
||||
const canvas = document.createElement('canvas') as HTMLCanvasElement;
|
||||
initCanvas();
|
||||
|
||||
function initCanvas() {
|
||||
canvas.width = detFeedShape;
|
||||
canvas.height = detFeedShape;
|
||||
canvas.style.position = 'absolute';
|
||||
canvas.style.top = '0';
|
||||
canvas.style.left = '0';
|
||||
canvas.style.zIndex = '-1';
|
||||
canvas.style.opacity = '0';
|
||||
document.body.appendChild(canvas);
|
||||
}
|
||||
|
||||
export async function load() {
|
||||
anchorResults = anchor.replace(/\s+/g, ',').split(',').map(item => +item);
|
||||
|
||||
detectRunner = new Runner({
|
||||
modelPath: 'https://paddlejs.bj.bcebos.com/models/fuse/gesture/gesture_det_fuse_activation/model.json',
|
||||
webglFeedProcess: true
|
||||
});
|
||||
const detectInit = detectRunner.init();
|
||||
|
||||
recRunner = new Runner({
|
||||
modelPath: 'https://paddlejs.bj.bcebos.com/models/fuse/gesture/gesture_rec_fuse_activation/model.json'
|
||||
});
|
||||
const recInit = recRunner.init();
|
||||
|
||||
WarpAffine.init({
|
||||
width: 224,
|
||||
height: 224
|
||||
});
|
||||
|
||||
return await Promise.all([detectInit, recInit]);
|
||||
}
|
||||
|
||||
export async function classify(image): Promise<{ box: string | number[][], type: string }> {
|
||||
canvas.getContext('2d')!.drawImage(image, 0, 0, detFeedShape, detFeedShape);
|
||||
const res = await detectRunner.predict(image);
|
||||
const post = new DetectProcess(res, canvas);
|
||||
const result = {
|
||||
box: '',
|
||||
type: ''
|
||||
};
|
||||
box = await post.outputBox(anchorResults);
|
||||
if (box) {
|
||||
// 手势框选位置
|
||||
result.box = box;
|
||||
const feed = await post.outputFeed(recRunner);
|
||||
const res2 = await recRunner.predictWithFeed(feed);
|
||||
const lmProcess = new LMProcess(res2);
|
||||
lmProcess.output();
|
||||
// 识别结果
|
||||
result.type = lmProcess.type || '';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
const VSHADER_SOURCE = `attribute vec4 a_Position;
|
||||
attribute vec2 a_TexCoord;
|
||||
uniform mat4 translation;
|
||||
varying vec2 v_TexCoord;
|
||||
void main() {
|
||||
gl_Position = translation * a_Position;
|
||||
v_TexCoord = a_TexCoord;
|
||||
}`;
|
||||
|
||||
const FSHADER_SOURCE = `precision mediump float;
|
||||
uniform sampler2D u_Sampler;
|
||||
varying vec2 v_TexCoord;
|
||||
void main() {
|
||||
gl_FragColor = texture2D(u_Sampler, v_TexCoord);
|
||||
}`;
|
||||
|
||||
let imgData = null;
|
||||
let mtrData: number[][] = null as unknown as number[][];
|
||||
let gl: any = null;
|
||||
|
||||
function init(output) {
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = output.width;
|
||||
canvas.height = output.height;
|
||||
|
||||
gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
|
||||
// 初始化之前先加载图片
|
||||
if (!initShaders(gl, VSHADER_SOURCE, FSHADER_SOURCE)) {
|
||||
throw new Error('initShaders false');
|
||||
}
|
||||
}
|
||||
function main(opt) {
|
||||
const {
|
||||
imageData,
|
||||
mtr,
|
||||
output
|
||||
} = opt;
|
||||
|
||||
mtrData = mtr;
|
||||
imgData = imageData;
|
||||
|
||||
gl.clearColor(0, 0, 0, 1);
|
||||
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
|
||||
const n = initVertexBuffers(gl);
|
||||
initTextures(gl, n, 0);
|
||||
const outputData = new Uint8Array(output.width * output.height * 4);
|
||||
gl.readPixels(0, 0, output.width, output.height, gl.RGBA, gl.UNSIGNED_BYTE, outputData);
|
||||
return outputData;
|
||||
}
|
||||
|
||||
function initVertexBuffers(gl) {
|
||||
// 顶点坐标和纹理图像坐标
|
||||
const vertices = new Float32Array([
|
||||
// webgl坐标，纹理坐标
|
||||
-1.0, 1.0, 0.0, 0.0, 1.0,
|
||||
-1.0, -1.0, 0.0, 0.0, 0.0,
|
||||
1.0, 1.0, 0.0, 1.0, 1.0,
|
||||
1.0, -1.0, 0.0, 1.0, 0.0
|
||||
]);
|
||||
|
||||
const FSIZE = vertices.BYTES_PER_ELEMENT;
|
||||
|
||||
const vertexBuffer = gl.createBuffer();
|
||||
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
|
||||
gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW);
|
||||
|
||||
const aPosition = gl.getAttribLocation(gl.program, 'a_Position');
|
||||
const aTexCoord = gl.getAttribLocation(gl.program, 'a_TexCoord');
|
||||
|
||||
const xformMatrix = new Float32Array([
|
||||
mtrData[0][0], mtrData[1][0], 0.0, 0.0,
|
||||
mtrData[0][1], mtrData[1][1], 0.0, 0.0,
|
||||
0.0, 0.0, 0.0, 1.0,
|
||||
mtrData[0][2], mtrData[1][2], 0.0, 1.0
|
||||
|
||||
]);
|
||||
|
||||
const translation = gl.getUniformLocation(gl.program, 'translation');
|
||||
gl.uniformMatrix4fv(translation, false, xformMatrix);
|
||||
|
||||
gl.vertexAttribPointer(aPosition, 3, gl.FLOAT, false, FSIZE * 5, 0);
|
||||
gl.enableVertexAttribArray(aPosition);
|
||||
|
||||
gl.vertexAttribPointer(aTexCoord, 2, gl.FLOAT, false, FSIZE * 5, FSIZE * 3);
|
||||
gl.enableVertexAttribArray(aTexCoord);
|
||||
|
||||
return 4;
|
||||
}
|
||||
|
||||
|
||||
function initTextures(gl, n, index) {
|
||||
// 创建纹理对象
|
||||
const texture = gl.createTexture();
|
||||
const uSampler = gl.getUniformLocation(gl.program, 'u_Sampler');
|
||||
|
||||
// WebGL纹理坐标中的纵轴方向和PNG，JPG等图片容器的Y轴方向是反的，所以先反转Y轴
|
||||
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1);
|
||||
|
||||
// 激活纹理单元，开启index号纹理单元
|
||||
gl.activeTexture(gl[`TEXTURE${index}`]);
|
||||
|
||||
// 绑定纹理对象
|
||||
gl.bindTexture(gl.TEXTURE_2D, texture);
|
||||
|
||||
// 配置纹理对象的参数
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
||||
|
||||
// 将纹理图像分配给纹理对象
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, imgData);
|
||||
|
||||
// 将纹理单元编号传给着色器
|
||||
gl.uniform1i(uSampler, index);
|
||||
|
||||
// 绘制
|
||||
gl.drawArrays(gl.TRIANGLE_STRIP, 0, n);
|
||||
}
|
||||
|
||||
function initShaders(gl, vshader, fshader) {
|
||||
const vertexShader = createShader(gl, vshader, gl.VERTEX_SHADER);
|
||||
const fragmentShader = createShader(gl, fshader, gl.FRAGMENT_SHADER);
|
||||
|
||||
const program = gl.createProgram();
|
||||
gl.attachShader(program, vertexShader);
|
||||
gl.attachShader(program, fragmentShader);
|
||||
|
||||
if (!program) {
|
||||
console.log('无法创建程序对象');
|
||||
return false;
|
||||
}
|
||||
gl.linkProgram(program);
|
||||
gl.useProgram(program);
|
||||
gl.program = program;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
function createShader(gl, sourceCode, type) {
|
||||
// Compiles either a shader of type gl.VERTEX_SHADER or gl.FRAGMENT_SHADER
|
||||
const shader = gl.createShader(type);
|
||||
gl.shaderSource(shader, sourceCode);
|
||||
gl.compileShader(shader);
|
||||
|
||||
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
||||
const info = gl.getShaderInfoLog(shader);
|
||||
throw Error('Could not compile WebGL program. \n\n' + info);
|
||||
}
|
||||
return shader;
|
||||
}
|
||||
|
||||
export default {
|
||||
main,
|
||||
init
|
||||
};
|
||||
Reference in New Issue
Block a user