FastDeploy/examples/application/js/package/packages/paddlejs-models/gesture/src/index.ts

/**
 * @file gesture model
 */

import { Runner } from '@paddlejs/paddlejs-core';
import '@paddlejs/paddlejs-backend-webgl';
import WarpAffine from './warpAffine';
import DetectProcess from './detectProcess';
import LMProcess from './LMProcess';
import anchor from './anchor_small.txt';

let box = null;
let detectRunner = null as Runner;
let recRunner = null as Runner;
let anchorResults = null;
const detFeedShape = 256;
const canvas = document.createElement('canvas') as HTMLCanvasElement;
initCanvas();

function initCanvas() {
    canvas.width = detFeedShape;
    canvas.height = detFeedShape;
    canvas.style.position = 'absolute';
    canvas.style.top = '0';
    canvas.style.left = '0';
    canvas.style.zIndex = '-1';
    canvas.style.opacity = '0';
    document.body.appendChild(canvas);
}

export async function load() {
    anchorResults = anchor.replace(/\s+/g, ',').split(',').map(item => +item);

    detectRunner = new Runner({
        modelPath: 'https://paddlejs.bj.bcebos.com/models/fuse/gesture/gesture_det_fuse_activation/model.json',
        webglFeedProcess: true
    });
    const detectInit = detectRunner.init();

    recRunner = new Runner({
        modelPath: 'https://paddlejs.bj.bcebos.com/models/fuse/gesture/gesture_rec_fuse_activation/model.json'
    });
    const recInit = recRunner.init();

    WarpAffine.init({
        width: 224,
        height: 224
    });

    return await Promise.all([detectInit, recInit]);
}

export async function classify(image): Promise<{ box: string | number[][], type: string }> {
    canvas.getContext('2d')!.drawImage(image, 0, 0, detFeedShape, detFeedShape);
    const res = await detectRunner.predict(image);
    const post = new DetectProcess(res, canvas);
    const result = {
        box: '',
        type: ''
    };
    box = await post.outputBox(anchorResults);
    if (box) {
        // 手势框选位置
        result.box = box;
        const feed = await post.outputFeed(recRunner);
        const res2 = await recRunner.predictWithFeed(feed);
        const lmProcess = new LMProcess(res2);
        lmProcess.output();
        // 识别结果
        result.type = lmProcess.type || '';
    }
    return result;
}