[Other] Refactor js submodule (#415)

* Refactor js submodule

* Remove change-log

* Update ocr module

* Update ocr-detection module

* Update ocr-detection module

* Remove change-log
This commit is contained in:
chenqianhe
2022-10-23 14:05:13 +08:00
committed by GitHub
parent 30971cf3fd
commit f2619b0546
273 changed files with 14697 additions and 5088 deletions

View File

@@ -0,0 +1,81 @@
/**
* @file seg or blur origin image
*/
function mainFunc({
out
}) {
const THRESHHOLD = 0.2;
return `
#define SIGMA SIGMA 3.0
#define BLUR_MSIZE 8
#define MSIZE 3
#define kernelSize 5.0
#define weight 1.0
uniform int type; // 0: blurBackground 1: drawHumanseg 2: drawMask
void main() {
vec2 outCoord = vCoord.xy;
outCoord.y = 1.0 - vCoord.y;
vec2 sourceTextureSize = vec2(${out.width_shape}, ${out.height_shape});
vec2 sourceTexelSize = 1.0 / sourceTextureSize;
float kernel[MSIZE]; // 3
kernel[0] = 0.12579369017522166;
kernel[1] = 0.13298;
kernel[2] = 0.12579369017522166;
float origin_alpha = 1.0 - TEXTURE2D(texture_origin, vec2(outCoord.x, outCoord.y) / 2.0).r;
vec4 counter = TEXTURE2D(texture_counter, outCoord.xy);
vec4 res = vec4(0.0);
if (type == 0) {
// Simple Cheap Box Blur
float pixelSizeX = 1.0 / float(${out.width_shape});
float pixelSizeY = 1.0 / float(${out.height_shape});
// Horizontal Blur
vec4 accumulation = vec4(0);
float weightsum = 0.0;
for (float i = -kernelSize; i <= kernelSize; i++){
accumulation += TEXTURE2D(texture_counter, outCoord.xy + vec2(i * pixelSizeX, 0.0)) * weight;
weightsum += weight;
}
// Vertical Blur
for (float i = -kernelSize; i <= kernelSize; i++){
accumulation += TEXTURE2D(texture_counter, outCoord.xy + vec2(0.0, i * pixelSizeY)) * weight;
weightsum += weight;
}
res = accumulation / weightsum;
if (origin_alpha > ${THRESHHOLD}) {
res = counter;
}
}
else if (type == 1) {
res = counter;
res.a = origin_alpha;
}
else if (type == 2) {
if (origin_alpha > ${THRESHHOLD}) {
res = vec4(1.0);
res.a = origin_alpha;
}
}
setPackedOutput(res);
}
`;
}
export default {
mainFunc,
textureFuncConf: {
origin: [],
counter: []
}
};

View File

@@ -0,0 +1,65 @@
/**
* @file add deal origin op
*/
import { Transformer, env, interfaces } from '@paddlejs/paddlejs-core';
const IMG_ORIGIN = 'image';
const FINAL_PACK_OP_NAME = 'fetch_pack';
const DEFAULT_WIDTH = 500;
const DEFAULT_HEIGHT = 280;
export default class DealOrigin extends Transformer {
private width;
private height;
constructor(width?: number, height?: number) {
super('DealOrigin');
this.width = width || DEFAULT_WIDTH;
this.height = height || DEFAULT_HEIGHT;
}
transform(...args: any) {
if (!env.get('webgl_gpu_pipeline')) {
return;
}
const [ops, vars] = args;
const fetchOp = ops.find(item => item.type === 'fetch');
const [inputName] = fetchOp.inputs.X;
const segImgOp = {
attrs: {},
inputs: {
X: [inputName],
Y: [IMG_ORIGIN]
},
outputs: {
Out: [FINAL_PACK_OP_NAME]
},
type: 'segImg',
isPacked: true,
bufferType: interfaces.BufferType.ColorBuffer,
uniform: {
type: {
type: '1i',
value: 0
}
}
};
const packOutVar = {
name: FINAL_PACK_OP_NAME,
shape: [1, 1, this.height, this.width],
persistable: false
};
fetchOp.inputs.X = [FINAL_PACK_OP_NAME];
ops.push(segImgOp);
if (vars instanceof Array) {
vars.push(...[packOutVar]);
}
else {
vars[FINAL_PACK_OP_NAME] = packOutVar;
}
}
}

View File

@@ -0,0 +1,217 @@
/**
* @file humanseg model
*/
import { Runner, env } from '@paddlejs/paddlejs-core';
import '@paddlejs/paddlejs-backend-webgl';
import WebGLImageFilter from '../thirdParty/webgl-image-filter';
let runner = null as Runner;
let inputElement: any = null;
let WIDTH = 398;
let HEIGHT = 224;
let backgroundSize: any = null;
const blurFilter = new WebGLImageFilter();
blurFilter.reset();
blurFilter.addFilter('blur', 10);
export async function load(needPreheat = true, enableLightModel = false, customModel = null) {
const modelpath = 'https://paddlejs.bj.bcebos.com/models/fuse/humanseg/humanseg_398x224_fuse_activation/model.json';
const lightModelPath = 'https://paddlejs.bj.bcebos.com/models/fuse/humanseg/humanseg_288x160_fuse_activation/model.json';
const path = customModel
? customModel
: enableLightModel ? lightModelPath : modelpath;
if (enableLightModel) {
WIDTH = 288;
HEIGHT = 160;
}
runner = new Runner({
modelPath: path,
needPreheat: needPreheat !== undefined ? needPreheat : true,
mean: [0.5, 0.5, 0.5],
std: [0.5, 0.5, 0.5],
webglFeedProcess: true
});
env.set('webgl_pack_channel', true);
env.set('webgl_pack_output', true);
await runner.init();
if (runner.feedShape) {
WIDTH = runner.feedShape.fw;
HEIGHT = runner.feedShape.fh;
}
}
export async function preheat() {
return await runner.preheat();
}
export async function getGrayValue(input: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement) {
inputElement = input;
const seg_values = await runner.predict(input);
backgroundSize = genBackgroundSize();
return {
width: WIDTH,
height: HEIGHT,
data: seg_values
};
}
function genBackgroundSize() {
// 缩放后的宽高
let sw = WIDTH;
let sh = HEIGHT;
const ratio = sw / sh;
const inputWidth = inputElement.naturalWidth || inputElement.width;
const inputHeight = inputElement.naturalHeight || inputElement.height;
let x = 0;
let y = 0;
let bx = 0;
let by = 0;
let bh = inputHeight;
let bw = inputWidth;
const origin_ratio = inputWidth / inputHeight;
// target的长宽比大些 就把原图的高变成target那么高
if (ratio / origin_ratio >= 1) {
sw = sh * origin_ratio;
x = Math.floor((WIDTH - sw) / 2);
bw = bh * ratio;
bx = Math.floor((bw - inputWidth) / 2);
}
// target的长宽比小些 就把原图的宽变成target那么宽
else {
sh = sw / origin_ratio;
y = Math.floor((HEIGHT - sh) / 2);
bh = bw / ratio;
by = Math.floor((bh - inputHeight) / 2);
}
return {
x,
y,
sw,
sh,
bx,
by,
bw,
bh
};
}
/**
* draw human seg
* @param {Array} seg_values seg values of the input image
* @param {HTMLCanvasElement} canvas the dest canvas draws the pixels
* @param {HTMLCanvasElement} backgroundCanvas the background canvas draws the pixels
*/
export function drawHumanSeg(
seg_values: number[],
canvas: HTMLCanvasElement,
backgroundCanvas?: HTMLCanvasElement | HTMLImageElement
) {
const inputWidth = inputElement.naturalWidth || inputElement.width;
const inputHeight = inputElement.naturalHeight || inputElement.height;
const ctx = canvas.getContext('2d') as CanvasRenderingContext2D;
canvas.width = WIDTH;
canvas.height = HEIGHT;
const tempCanvas = document.createElement('canvas') as HTMLCanvasElement;
const tempContext = tempCanvas.getContext('2d') as CanvasRenderingContext2D;
tempCanvas.width = WIDTH;
tempCanvas.height = HEIGHT;
const tempScaleData = ctx.getImageData(0, 0, WIDTH, HEIGHT);
tempContext.drawImage(inputElement, backgroundSize.x, backgroundSize.y, backgroundSize.sw, backgroundSize.sh);
const originImageData = tempContext.getImageData(0, 0, WIDTH, HEIGHT);
for (let i = 0; i < WIDTH * HEIGHT; i++) {
if (seg_values[i + WIDTH * HEIGHT] * 255 > 100) {
tempScaleData.data[i * 4] = originImageData.data[i * 4];
tempScaleData.data[i * 4 + 1] = originImageData.data[i * 4 + 1];
tempScaleData.data[i * 4 + 2] = originImageData.data[i * 4 + 2];
tempScaleData.data[i * 4 + 3] = seg_values[i + WIDTH * HEIGHT] * 255;
}
}
tempContext.putImageData(tempScaleData, 0, 0);
canvas.width = inputWidth;
canvas.height = inputHeight;
backgroundCanvas
&& ctx.drawImage(backgroundCanvas, -backgroundSize.bx, -backgroundSize.by, backgroundSize.bw, backgroundSize.bh);
ctx.drawImage(tempCanvas, -backgroundSize.bx, -backgroundSize.by, backgroundSize.bw, backgroundSize.bh);
}
/**
* draw human seg
* @param {HTMLCanvasElement} canvas the dest canvas draws the pixels
* @param {Array} seg_values seg_values of the input image
*/
export function blurBackground(seg_values: number[], dest_canvas) {
const inputWidth = inputElement.naturalWidth || inputElement.width;
const inputHeight = inputElement.naturalHeight || inputElement.height;
const tempCanvas = document.createElement('canvas') as HTMLCanvasElement;
const tempContext = tempCanvas.getContext('2d') as CanvasRenderingContext2D;
tempCanvas.width = WIDTH;
tempCanvas.height = HEIGHT;
const dest_ctx = dest_canvas.getContext('2d') as CanvasRenderingContext2D;
dest_canvas.width = inputWidth;
dest_canvas.height = inputHeight;
const tempScaleData = tempContext.getImageData(0, 0, WIDTH, HEIGHT);
tempContext.drawImage(inputElement, backgroundSize.x, backgroundSize.y, backgroundSize.sw, backgroundSize.sh);
const originImageData = tempContext.getImageData(0, 0, WIDTH, HEIGHT);
blurFilter.dispose();
const blurCanvas = blurFilter.apply(tempCanvas);
for (let i = 0; i < WIDTH * HEIGHT; i++) {
if (seg_values[i + WIDTH * HEIGHT] * 255 > 150) {
tempScaleData.data[i * 4] = originImageData.data[i * 4];
tempScaleData.data[i * 4 + 1] = originImageData.data[i * 4 + 1];
tempScaleData.data[i * 4 + 2] = originImageData.data[i * 4 + 2];
tempScaleData.data[i * 4 + 3] = seg_values[i + WIDTH * HEIGHT] * 255;
}
}
tempContext.putImageData(tempScaleData, 0, 0);
dest_ctx.drawImage(blurCanvas, -backgroundSize.bx, -backgroundSize.by, backgroundSize.bw, backgroundSize.bh);
dest_ctx.drawImage(tempCanvas, -backgroundSize.bx, -backgroundSize.by, backgroundSize.bw, backgroundSize.bh);
}
/**
* draw mask without human
* @param {Array} seg_values seg_values of the input image
* @param {HTMLCanvasElement} dest the dest canvas draws the pixels
* @param {HTMLCanvasElement} canvas background canvas
*/
export function drawMask(seg_values: number[], dest: HTMLCanvasElement, canvas: HTMLCanvasElement) {
const tempCanvas = document.createElement('canvas') as HTMLCanvasElement;
const tempContext = tempCanvas.getContext('2d') as CanvasRenderingContext2D;
tempCanvas.width = WIDTH;
tempCanvas.height = HEIGHT;
tempContext.drawImage(canvas, 0, 0, WIDTH, HEIGHT);
const dest_ctx = dest.getContext('2d') as CanvasRenderingContext2D;
dest.width = WIDTH;
dest.height = HEIGHT;
const tempScaleData = tempContext.getImageData(0, 0, WIDTH, HEIGHT);
for (let i = 0; i < WIDTH * HEIGHT; i++) {
if (seg_values[i + WIDTH * HEIGHT] * 255 > 150) {
tempScaleData.data[i * 4 + 3] = seg_values[i] * 255;
}
}
tempContext.putImageData(tempScaleData, 0, 0);
dest_ctx.drawImage(tempCanvas, 0, 0, WIDTH, HEIGHT);
}

View File

@@ -0,0 +1,219 @@
/**
* @file humanseg gpu pipeline
*/
import { Runner, env, registerOp } from '@paddlejs/paddlejs-core';
import { GLHelper } from '@paddlejs/paddlejs-backend-webgl';
import segImg from './customOp/segImg';
import AppendDealOriginOpToNN from './customTransformer/appendCustomOpToNN';
interface LoadOptions {
needPreheat?: boolean,
enableLightModel?: boolean,
canvasWidth?: number,
canvasHeight?: number
}
let runner = null as Runner;
const WIDTH = 398;
const HEIGHT = 224;
function registerCustomOp() {
registerOp(segImg, 'segImg');
}
registerCustomOp();
const WEBGL_ATTRIBUTES = Object.assign({}, GLHelper.WEBGL_ATTRIBUTES, {
alpha: true
});
function createWebglContext(canvas: HTMLCanvasElement) {
let gl = canvas.getContext('webgl2', WEBGL_ATTRIBUTES) as WebGLRenderingContext | null;
if (gl) {
env.set('webglVersion', 2);
}
else {
env.set('webglVersion', 1);
gl = (canvas.getContext('webgl', WEBGL_ATTRIBUTES)
|| canvas.getContext('experimental-webgl', WEBGL_ATTRIBUTES)) as WebGLRenderingContext;
}
return gl as WebGLRenderingContext;
}
const renderCanvas = document.createElement('canvas');
renderCanvas.width = 500;
renderCanvas.height = 280;
const gl = createWebglContext(renderCanvas);
let segImgOp = null;
export async function load(options: LoadOptions = {
needPreheat: true,
enableLightModel: false,
canvasWidth: 500,
canvasHeight: 280
}) {
const modelPath = 'https://paddlejs.cdn.bcebos.com/models/humansegv2/model.json';
runner = new Runner({
modelPath: modelPath,
needPreheat: options.needPreheat !== undefined ? options.needPreheat : true,
feedShape: {
fw: WIDTH,
fh: HEIGHT
},
fill: '#fff',
mean: [0.5, 0.5, 0.5],
std: [0.5, 0.5, 0.5],
plugins: {
preTransforms: [new AppendDealOriginOpToNN(options.canvasWidth, options.canvasHeight)]
}
});
GLHelper.setWebGLRenderingContext(gl);
env.set('webgl_pack_channel', true);
env.set('webgl_gpu_pipeline', true);
env.set('webgl_force_half_float_texture', true);
await runner.init();
}
export async function preheat() {
return await runner.preheat();
}
/**
* draw human seg
* @param {HTMLImageElement | HTMLVideoElement | HTMLCanvasElement} input the input image
* @param {HTMLCanvasElement} canvas the dest canvas draws the pixels
* @param {HTMLCanvasElement} back background canvas
*/
export async function drawHumanSeg(
input: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement,
canvas: HTMLCanvasElement,
back?: HTMLCanvasElement
) {
if (!segImgOp) {
segImgOp = runner.weightMap[runner.weightMap.length - 1].opData;
}
// todo: 底层库更新类型声明后优化这里的 ignore
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
segImgOp.uniform.type.value = 1;
await runner.predict(input);
const backgroundSize = genBackgroundSize(input);
canvas.width = input.width;
canvas.height = input.height;
const destCtx = canvas.getContext('2d');
if (back) {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
destCtx.drawImage(back, -backgroundSize.bx, -backgroundSize.by, backgroundSize.bw, backgroundSize.bh);
}
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
destCtx.drawImage(gl.canvas, -backgroundSize.bx, -backgroundSize.by, backgroundSize.bw, backgroundSize.bh);
}
/**
* draw human seg
* @param {HTMLImageElement | HTMLVideoElement | HTMLCanvasElement} input the input image
* @param {HTMLCanvasElement} canvas the dest canvas draws the pixels
*/
export async function blurBackground(
input: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement,
canvas: HTMLCanvasElement
) {
if (!segImgOp) {
segImgOp = runner.weightMap[runner.weightMap.length - 1].opData;
}
// todo: 底层库更新类型声明后优化这里的 ignore
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
segImgOp.uniform.type.value = 0;
await runner.predict(input);
canvas.width = input.width;
canvas.height = input.height;
const backgroundSize = genBackgroundSize(input);
const destCtx = canvas.getContext('2d');
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
destCtx.drawImage(gl.canvas, -backgroundSize.bx, -backgroundSize.by, backgroundSize.bw, backgroundSize.bh);
}
/**
* draw human mask
* @param {HTMLImageElement | HTMLVideoElement | HTMLCanvasElement} input the input image
* @param {HTMLCanvasElement} canvas the dest canvas draws the pixels
* @param {HTMLCanvasElement} back background canvas
*/
export async function drawMask(
input: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement,
canvas: HTMLCanvasElement,
back: HTMLCanvasElement
) {
if (!segImgOp) {
segImgOp = runner.weightMap[runner.weightMap.length - 1].opData;
}
// todo: 底层库更新类型声明后优化这里的 ignore
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
segImgOp.uniform.type.value = 2;
await runner.predict(input);
canvas.width = input.width;
canvas.height = input.height;
const backgroundSize = genBackgroundSize(input);
const destCtx = canvas.getContext('2d');
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
destCtx.drawImage(back, -backgroundSize.bx, -backgroundSize.by, backgroundSize.bw, backgroundSize.bh);
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
destCtx.drawImage(gl.canvas, -backgroundSize.bx, -backgroundSize.by, backgroundSize.bw, backgroundSize.bh);
}
function genBackgroundSize(inputElement) {
// 缩放后的宽高
let sw = WIDTH;
let sh = HEIGHT;
const ratio = sw / sh;
const inputWidth = inputElement.naturalWidth || inputElement.width;
const inputHeight = inputElement.naturalHeight || inputElement.height;
let x = 0;
let y = 0;
let bx = 0;
let by = 0;
let bh = inputHeight;
let bw = inputWidth;
const origin_ratio = inputWidth / inputHeight;
// target的长宽比大些 就把原图的高变成target那么高
if (ratio / origin_ratio >= 1) {
sw = sh * origin_ratio;
x = Math.floor((WIDTH - sw) / 2);
bw = bh * ratio;
bx = Math.floor((bw - inputWidth) / 2);
}
// target的长宽比小些 就把原图的宽变成target那么宽
else {
sh = sw / origin_ratio;
y = Math.floor((HEIGHT - sh) / 2);
bh = bw / ratio;
by = Math.floor((bh - inputHeight) / 2);
}
return {
x,
y,
sw,
sh,
bx,
by,
bw,
bh
};
}