commit ba6e1afb77e2d36d17b0a9bc4bb630f9b71a7409 Author: esimov Date: Thu Apr 9 12:02:49 2020 +0300 First commit diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7a506e9 --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +ifeq ($(OS),Windows_NT) + BROWSER = start +else + UNAME := $(shell uname -s) + ifeq ($(UNAME), Linux) + BROWSER = xdg-open + endif + ifeq ($(UNAME), Darwin) + BROWSER = open + endif +endif + +.PHONY: all clean serve + +all: wasm serve + +demo: masquerade serve + +masquerade: + cp -f "$$(go env GOROOT)/misc/wasm/wasm_exec.js" ./js/ + GOOS=js GOARCH=wasm go build -o lib.wasm masquerade.go + +serve: + $(BROWSER) 'http://localhost:5000' + serve + +clean: + rm -f *.wasm + +debug: + @echo $(UNAME) diff --git a/css/style.css b/css/style.css new file mode 100644 index 0000000..89a3861 --- /dev/null +++ b/css/style.css @@ -0,0 +1,11 @@ +html, body { + background: #000; + margin: 0; + padding: 0; +} + +#canvas { + position: absolute; + left: 50%; top: 50%; + transform: translate(-50%, -50%); +} \ No newline at end of file diff --git a/detector/detector.go b/detector/detector.go new file mode 100644 index 0000000..da59136 --- /dev/null +++ b/detector/detector.go @@ -0,0 +1,197 @@ +package detector + +import ( + "errors" + + pigo "github.com/esimov/pigo/core" +) + +// FlpCascade holds the binary representation of the facial landmark points cascade files +type FlpCascade struct { + *pigo.PuplocCascade + error +} + +var ( + cascade []byte + puplocCascade []byte + faceClassifier *pigo.Pigo + puplocClassifier *pigo.PuplocCascade + flpcs map[string][]*FlpCascade + imgParams *pigo.ImageParams + err error +) + +var ( + eyeCascades = []string{"lp46", "lp44", "lp42", "lp38", "lp312"} + mouthCascade = []string{"lp93", "lp84", "lp82", "lp81"} +) + +// UnpackCascades unpack all of used cascade files. +func (d *Detector) UnpackCascades() error { + p := pigo.NewPigo() + + cascade, err = d.FetchCascade("https://raw.githubusercontent.com/esimov/pigo/master/cascade/facefinder") + if err != nil { + return errors.New("error reading the facefinder cascade file") + } + // Unpack the binary file. This will return the number of cascade trees, + // the tree depth, the threshold and the prediction from tree's leaf nodes. + faceClassifier, err = p.Unpack(cascade) + if err != nil { + return errors.New("error unpacking the facefinder cascade file") + } + + plc := pigo.NewPuplocCascade() + + puplocCascade, err = d.FetchCascade("https://raw.githubusercontent.com/esimov/pigo/master/cascade/puploc") + if err != nil { + return errors.New("error reading the puploc cascade file") + } + + puplocClassifier, err = plc.UnpackCascade(puplocCascade) + if err != nil { + return errors.New("error unpacking the puploc cascade file") + } + + flpcs, err = d.parseFlpCascades("https://raw.githubusercontent.com/esimov/pigo/master/cascade/lps/") + if err != nil { + return errors.New("error unpacking the facial landmark points detection cascades") + } + return nil +} + +// DetectFaces runs the cluster detection over the webcam frame +// received as a pixel array and returns the detected faces. +func (d *Detector) DetectFaces(pixels []uint8, width, height int) [][]int { + results := d.clusterDetection(pixels, width, height) + dets := make([][]int, len(results)) + + for i := 0; i < len(results); i++ { + dets[i] = append(dets[i], results[i].Row, results[i].Col, results[i].Scale, int(results[i].Q)) + } + return dets +} + +// DetectLeftPupil detects the left pupil +func (d *Detector) DetectLeftPupil(results []int) *pigo.Puploc { + puploc := &pigo.Puploc{ + Row: results[0] - int(0.085*float32(results[2])), + Col: results[1] - int(0.185*float32(results[2])), + Scale: float32(results[2]) * 0.4, + Perturbs: 63, + } + leftEye := puplocClassifier.RunDetector(*puploc, *imgParams, 0.0, false) + if leftEye.Row > 0 && leftEye.Col > 0 { + return leftEye + } + return nil +} + +// DetectRightPupil detects the right pupil +func (d *Detector) DetectRightPupil(results []int) *pigo.Puploc { + puploc := &pigo.Puploc{ + Row: results[0] - int(0.085*float32(results[2])), + Col: results[1] + int(0.185*float32(results[2])), + Scale: float32(results[2]) * 0.4, + Perturbs: 63, + } + rightEye := puplocClassifier.RunDetector(*puploc, *imgParams, 0.0, false) + if rightEye.Row > 0 && rightEye.Col > 0 { + return rightEye + } + return nil +} + +// DetectLandmarkPoints detects the landmark points +func (d *Detector) DetectLandmarkPoints(leftEye, rightEye *pigo.Puploc) [][]int { + var ( + det = make([][]int, 15) + idx int + ) + + for _, eye := range eyeCascades { + for _, flpc := range flpcs[eye] { + flp := flpc.FindLandmarkPoints(leftEye, rightEye, *imgParams, 63, false) + if flp.Row > 0 && flp.Col > 0 { + det[idx] = append(det[idx], flp.Col, flp.Row, int(flp.Scale)) + } + idx++ + + flp = flpc.FindLandmarkPoints(leftEye, rightEye, *imgParams, 63, true) + if flp.Row > 0 && flp.Col > 0 { + det[idx] = append(det[idx], flp.Col, flp.Row, int(flp.Scale)) + } + idx++ + } + } + + for _, mouth := range mouthCascade { + for _, flpc := range flpcs[mouth] { + flp := flpc.FindLandmarkPoints(leftEye, rightEye, *imgParams, 63, false) + if flp.Row > 0 && flp.Col > 0 { + det[idx] = append(det[idx], flp.Col, flp.Row, int(flp.Scale)) + } + idx++ + } + } + flp := flpcs["lp84"][0].FindLandmarkPoints(leftEye, rightEye, *imgParams, 63, true) + if flp.Row > 0 && flp.Col > 0 { + det[idx] = append(det[idx], flp.Col, flp.Row, int(flp.Scale)) + } + return det +} + +func (d *Detector) DetectMouthPoints(leftEye, rightEye *pigo.Puploc) [][]int { + flp1 := flpcs["lp84"][0].FindLandmarkPoints(leftEye, rightEye, *imgParams, 63, false) + flp2 := flpcs["lp84"][0].FindLandmarkPoints(leftEye, rightEye, *imgParams, 63, true) + return [][]int{ + []int{flp1.Col, flp1.Row, int(flp1.Scale)}, + []int{flp2.Col, flp2.Row, int(flp2.Scale)}, + } +} + +// clusterDetection runs Pigo face detector core methods +// and returns a cluster with the detected faces coordinates. +func (d *Detector) clusterDetection(pixels []uint8, width, height int) []pigo.Detection { + imgParams = &pigo.ImageParams{ + Pixels: pixels, + Rows: width, + Cols: height, + Dim: height, + } + cParams := pigo.CascadeParams{ + MinSize: 200, + MaxSize: 720, + ShiftFactor: 0.1, + ScaleFactor: 1.02, + ImageParams: *imgParams, + } + + // Run the classifier over the obtained leaf nodes and return the detection results. + // The result contains quadruplets representing the row, column, scale and detection score. + dets := faceClassifier.RunCascade(cParams, 0.0) + + // Calculate the intersection over union (IoU) of two clusters. + dets = faceClassifier.ClusterDetections(dets, 0.1) + + return dets +} + +// parseFlpCascades reads the facial landmark points cascades from the provided url. +func (d *Detector) parseFlpCascades(path string) (map[string][]*FlpCascade, error) { + cascades := append(eyeCascades, mouthCascade...) + flpcs := make(map[string][]*FlpCascade) + + pl := pigo.NewPuplocCascade() + + for _, cascade := range cascades { + puplocCascade, err = d.FetchCascade(path + cascade) + if err != nil { + d.Log("Error reading the cascade file: %v", err) + } + flpc, err := pl.UnpackCascade(puplocCascade) + flpcs[cascade] = append(flpcs[cascade], &FlpCascade{flpc, err}) + } + return flpcs, err +} diff --git a/detector/fetch.go b/detector/fetch.go new file mode 100644 index 0000000..0a3a7ab --- /dev/null +++ b/detector/fetch.go @@ -0,0 +1,69 @@ +package detector + +import ( + "fmt" + "syscall/js" +) + +// Detector struct holds the main components of the fetching operation. +type Detector struct { + respChan chan []uint8 + errChan chan error + done chan struct{} + + window js.Value +} + +// NewDetector initializes a new constructor function. +func NewDetector() *Detector { + var d Detector + d.window = js.Global() + + return &d +} + +// FetchCascade retrive the cascade file trough a JS http connection. +// It should return the binary data as uint8 integers or err in case of an error. +func (d *Detector) FetchCascade(url string) ([]byte, error) { + d.respChan = make(chan []uint8) + d.errChan = make(chan error) + + promise := js.Global().Call("fetch", url) + success := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + response := args[0] + response.Call("arrayBuffer").Call("then", js.FuncOf(func(this js.Value, args []js.Value) interface{} { + go func() { + buffer := args[0] + uint8Array := js.Global().Get("Uint8Array").New(buffer) + + jsbuf := make([]byte, uint8Array.Get("length").Int()) + js.CopyBytesToGo(jsbuf, uint8Array) + d.respChan <- jsbuf + }() + return nil + })) + return nil + }) + + failure := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + go func() { + err := fmt.Errorf("unable to fetch the cascade file: %s", args[0].String()) + d.errChan <- err + }() + return nil + }) + + promise.Call("then", success, failure) + + select { + case resp := <-d.respChan: + return resp, nil + case err := <-d.errChan: + return nil, err + } +} + +// Log calls the `console.log` Javascript function +func (d *Detector) Log(args ...interface{}) { + d.window.Get("console").Call("log", args...) +} diff --git a/images/carnival.png b/images/carnival.png new file mode 100644 index 0000000..aa61479 Binary files /dev/null and b/images/carnival.png differ diff --git a/images/carnival2.png b/images/carnival2.png new file mode 100644 index 0000000..a7be558 Binary files /dev/null and b/images/carnival2.png differ diff --git a/images/facemask.png b/images/facemask.png new file mode 100644 index 0000000..1342598 Binary files /dev/null and b/images/facemask.png differ diff --git a/images/facemask2.png b/images/facemask2.png new file mode 100644 index 0000000..ef21e96 Binary files /dev/null and b/images/facemask2.png differ diff --git a/images/funny-surgical-mask-mustache.png b/images/funny-surgical-mask-mustache.png new file mode 100644 index 0000000..71f69e8 Binary files /dev/null and b/images/funny-surgical-mask-mustache.png differ diff --git a/images/neon-disco.png b/images/neon-disco.png new file mode 100644 index 0000000..89f2d2a Binary files /dev/null and b/images/neon-disco.png differ diff --git a/images/neon-green.png b/images/neon-green.png new file mode 100644 index 0000000..7323aa7 Binary files /dev/null and b/images/neon-green.png differ diff --git a/images/neon-yellow.png b/images/neon-yellow.png new file mode 100644 index 0000000..5e87046 Binary files /dev/null and b/images/neon-yellow.png differ diff --git a/images/sunglasses.png b/images/sunglasses.png new file mode 100644 index 0000000..427c0f2 Binary files /dev/null and b/images/sunglasses.png differ diff --git a/index.html b/index.html new file mode 100644 index 0000000..bef05fa --- /dev/null +++ b/index.html @@ -0,0 +1,33 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/js/stats.min.js b/js/stats.min.js new file mode 100644 index 0000000..ce648c0 --- /dev/null +++ b/js/stats.min.js @@ -0,0 +1,5 @@ +// stats.js - http://github.com/mrdoob/stats.js +(function(f,e){"object"===typeof exports&&"undefined"!==typeof module?module.exports=e():"function"===typeof define&&define.amd?define(e):f.Stats=e()})(this,function(){var f=function(){function e(a){c.appendChild(a.dom);return a}function u(a){for(var d=0;d=g+1E3&&(r.update(1E3*a/(c-g),100),g=c,a=0,t)){var d=performance.memory;t.update(d.usedJSHeapSize/ + 1048576,d.jsHeapSizeLimit/1048576)}return c},update:function(){k=this.end()},domElement:c,setMode:u}};f.Panel=function(e,f,l){var c=Infinity,k=0,g=Math.round,a=g(window.devicePixelRatio||1),r=80*a,h=48*a,t=3*a,v=2*a,d=3*a,m=15*a,n=74*a,p=30*a,q=document.createElement("canvas");q.width=r;q.height=h;q.style.cssText="width:80px;height:48px";var b=q.getContext("2d");b.font="bold "+9*a+"px Helvetica,Arial,sans-serif";b.textBaseline="top";b.fillStyle=l;b.fillRect(0,0,r,h);b.fillStyle=f;b.fillText(e,t,v); + b.fillRect(d,m,n,p);b.fillStyle=l;b.globalAlpha=.9;b.fillRect(d,m,n,p);return{dom:q,update:function(h,w){c=Math.min(c,h);k=Math.max(k,h);b.fillStyle=l;b.globalAlpha=1;b.fillRect(0,0,r,m);b.fillStyle=f;b.fillText(g(h)+" "+e+" ("+g(c)+"-"+g(k)+")",t,v);b.drawImage(q,d+a,m,n-a,p,d,m,n-a,p);b.fillRect(d+n-a,m,a,p);b.fillStyle=l;b.globalAlpha=.9;b.fillRect(d+n-a,m,a,g((1-h/w)*p))}}};return f}); \ No newline at end of file diff --git a/js/wasm_exec.js b/js/wasm_exec.js new file mode 100644 index 0000000..a54bb9a --- /dev/null +++ b/js/wasm_exec.js @@ -0,0 +1,533 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +(() => { + // Map multiple JavaScript environments to a single common API, + // preferring web standards over Node.js API. + // + // Environments considered: + // - Browsers + // - Node.js + // - Electron + // - Parcel + + if (typeof global !== "undefined") { + // global already exists + } else if (typeof window !== "undefined") { + window.global = window; + } else if (typeof self !== "undefined") { + self.global = self; + } else { + throw new Error("cannot export Go (neither global, window nor self is defined)"); + } + + if (!global.require && typeof require !== "undefined") { + global.require = require; + } + + if (!global.fs && global.require) { + global.fs = require("fs"); + } + + if (!global.fs) { + let outputBuf = ""; + global.fs = { + constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused + writeSync(fd, buf) { + outputBuf += decoder.decode(buf); + const nl = outputBuf.lastIndexOf("\n"); + if (nl != -1) { + console.log(outputBuf.substr(0, nl)); + outputBuf = outputBuf.substr(nl + 1); + } + return buf.length; + }, + write(fd, buf, offset, length, position, callback) { + if (offset !== 0 || length !== buf.length || position !== null) { + throw new Error("not implemented"); + } + const n = this.writeSync(fd, buf); + callback(null, n); + }, + open(path, flags, mode, callback) { + const err = new Error("not implemented"); + err.code = "ENOSYS"; + callback(err); + }, + read(fd, buffer, offset, length, position, callback) { + const err = new Error("not implemented"); + err.code = "ENOSYS"; + callback(err); + }, + fsync(fd, callback) { + callback(null); + }, + }; + } + + if (!global.crypto) { + const nodeCrypto = require("crypto"); + global.crypto = { + getRandomValues(b) { + nodeCrypto.randomFillSync(b); + }, + }; + } + + if (!global.performance) { + global.performance = { + now() { + const [sec, nsec] = process.hrtime(); + return sec * 1000 + nsec / 1000000; + }, + }; + } + + if (!global.TextEncoder) { + global.TextEncoder = require("util").TextEncoder; + } + + if (!global.TextDecoder) { + global.TextDecoder = require("util").TextDecoder; + } + + // End of polyfills for common API. + + const encoder = new TextEncoder("utf-8"); + const decoder = new TextDecoder("utf-8"); + + global.Go = class { + constructor() { + this.argv = ["js"]; + this.env = {}; + this.exit = (code) => { + if (code !== 0) { + console.warn("exit code:", code); + } + }; + this._exitPromise = new Promise((resolve) => { + this._resolveExitPromise = resolve; + }); + this._pendingEvent = null; + this._scheduledTimeouts = new Map(); + this._nextCallbackTimeoutID = 1; + + const mem = () => { + // The buffer may change when requesting more memory. + return new DataView(this._inst.exports.mem.buffer); + } + + const setInt64 = (addr, v) => { + mem().setUint32(addr + 0, v, true); + mem().setUint32(addr + 4, Math.floor(v / 4294967296), true); + } + + const getInt64 = (addr) => { + const low = mem().getUint32(addr + 0, true); + const high = mem().getInt32(addr + 4, true); + return low + high * 4294967296; + } + + const loadValue = (addr) => { + const f = mem().getFloat64(addr, true); + if (f === 0) { + return undefined; + } + if (!isNaN(f)) { + return f; + } + + const id = mem().getUint32(addr, true); + return this._values[id]; + } + + const storeValue = (addr, v) => { + const nanHead = 0x7FF80000; + + if (typeof v === "number") { + if (isNaN(v)) { + mem().setUint32(addr + 4, nanHead, true); + mem().setUint32(addr, 0, true); + return; + } + if (v === 0) { + mem().setUint32(addr + 4, nanHead, true); + mem().setUint32(addr, 1, true); + return; + } + mem().setFloat64(addr, v, true); + return; + } + + switch (v) { + case undefined: + mem().setFloat64(addr, 0, true); + return; + case null: + mem().setUint32(addr + 4, nanHead, true); + mem().setUint32(addr, 2, true); + return; + case true: + mem().setUint32(addr + 4, nanHead, true); + mem().setUint32(addr, 3, true); + return; + case false: + mem().setUint32(addr + 4, nanHead, true); + mem().setUint32(addr, 4, true); + return; + } + + let ref = this._refs.get(v); + if (ref === undefined) { + ref = this._values.length; + this._values.push(v); + this._refs.set(v, ref); + } + let typeFlag = 0; + switch (typeof v) { + case "string": + typeFlag = 1; + break; + case "symbol": + typeFlag = 2; + break; + case "function": + typeFlag = 3; + break; + } + mem().setUint32(addr + 4, nanHead | typeFlag, true); + mem().setUint32(addr, ref, true); + } + + const loadSlice = (addr) => { + const array = getInt64(addr + 0); + const len = getInt64(addr + 8); + return new Uint8Array(this._inst.exports.mem.buffer, array, len); + } + + const loadSliceOfValues = (addr) => { + const array = getInt64(addr + 0); + const len = getInt64(addr + 8); + const a = new Array(len); + for (let i = 0; i < len; i++) { + a[i] = loadValue(array + i * 8); + } + return a; + } + + const loadString = (addr) => { + const saddr = getInt64(addr + 0); + const len = getInt64(addr + 8); + return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len)); + } + + const timeOrigin = Date.now() - performance.now(); + this.importObject = { + go: { + // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters) + // may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported + // function. A goroutine can switch to a new stack if the current stack is too small (see morestack function). + // This changes the SP, thus we have to update the SP used by the imported function. + + // func wasmExit(code int32) + "runtime.wasmExit": (sp) => { + const code = mem().getInt32(sp + 8, true); + this.exited = true; + delete this._inst; + delete this._values; + delete this._refs; + this.exit(code); + }, + + // func wasmWrite(fd uintptr, p unsafe.Pointer, n int32) + "runtime.wasmWrite": (sp) => { + const fd = getInt64(sp + 8); + const p = getInt64(sp + 16); + const n = mem().getInt32(sp + 24, true); + fs.writeSync(fd, new Uint8Array(this._inst.exports.mem.buffer, p, n)); + }, + + // func nanotime() int64 + "runtime.nanotime": (sp) => { + setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000); + }, + + // func walltime() (sec int64, nsec int32) + "runtime.walltime": (sp) => { + const msec = (new Date).getTime(); + setInt64(sp + 8, msec / 1000); + mem().setInt32(sp + 16, (msec % 1000) * 1000000, true); + }, + + // func scheduleTimeoutEvent(delay int64) int32 + "runtime.scheduleTimeoutEvent": (sp) => { + const id = this._nextCallbackTimeoutID; + this._nextCallbackTimeoutID++; + this._scheduledTimeouts.set(id, setTimeout( + () => { + this._resume(); + while (this._scheduledTimeouts.has(id)) { + // for some reason Go failed to register the timeout event, log and try again + // (temporary workaround for https://github.com/golang/go/issues/28975) + console.warn("scheduleTimeoutEvent: missed timeout event"); + this._resume(); + } + }, + getInt64(sp + 8) + 1, // setTimeout has been seen to fire up to 1 millisecond early + )); + mem().setInt32(sp + 16, id, true); + }, + + // func clearTimeoutEvent(id int32) + "runtime.clearTimeoutEvent": (sp) => { + const id = mem().getInt32(sp + 8, true); + clearTimeout(this._scheduledTimeouts.get(id)); + this._scheduledTimeouts.delete(id); + }, + + // func getRandomData(r []byte) + "runtime.getRandomData": (sp) => { + crypto.getRandomValues(loadSlice(sp + 8)); + }, + + // func stringVal(value string) ref + "syscall/js.stringVal": (sp) => { + storeValue(sp + 24, loadString(sp + 8)); + }, + + // func valueGet(v ref, p string) ref + "syscall/js.valueGet": (sp) => { + const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16)); + sp = this._inst.exports.getsp(); // see comment above + storeValue(sp + 32, result); + }, + + // func valueSet(v ref, p string, x ref) + "syscall/js.valueSet": (sp) => { + Reflect.set(loadValue(sp + 8), loadString(sp + 16), loadValue(sp + 32)); + }, + + // func valueIndex(v ref, i int) ref + "syscall/js.valueIndex": (sp) => { + storeValue(sp + 24, Reflect.get(loadValue(sp + 8), getInt64(sp + 16))); + }, + + // valueSetIndex(v ref, i int, x ref) + "syscall/js.valueSetIndex": (sp) => { + Reflect.set(loadValue(sp + 8), getInt64(sp + 16), loadValue(sp + 24)); + }, + + // func valueCall(v ref, m string, args []ref) (ref, bool) + "syscall/js.valueCall": (sp) => { + try { + const v = loadValue(sp + 8); + const m = Reflect.get(v, loadString(sp + 16)); + const args = loadSliceOfValues(sp + 32); + const result = Reflect.apply(m, v, args); + sp = this._inst.exports.getsp(); // see comment above + storeValue(sp + 56, result); + mem().setUint8(sp + 64, 1); + } catch (err) { + storeValue(sp + 56, err); + mem().setUint8(sp + 64, 0); + } + }, + + // func valueInvoke(v ref, args []ref) (ref, bool) + "syscall/js.valueInvoke": (sp) => { + try { + const v = loadValue(sp + 8); + const args = loadSliceOfValues(sp + 16); + const result = Reflect.apply(v, undefined, args); + sp = this._inst.exports.getsp(); // see comment above + storeValue(sp + 40, result); + mem().setUint8(sp + 48, 1); + } catch (err) { + storeValue(sp + 40, err); + mem().setUint8(sp + 48, 0); + } + }, + + // func valueNew(v ref, args []ref) (ref, bool) + "syscall/js.valueNew": (sp) => { + try { + const v = loadValue(sp + 8); + const args = loadSliceOfValues(sp + 16); + const result = Reflect.construct(v, args); + sp = this._inst.exports.getsp(); // see comment above + storeValue(sp + 40, result); + mem().setUint8(sp + 48, 1); + } catch (err) { + storeValue(sp + 40, err); + mem().setUint8(sp + 48, 0); + } + }, + + // func valueLength(v ref) int + "syscall/js.valueLength": (sp) => { + setInt64(sp + 16, parseInt(loadValue(sp + 8).length)); + }, + + // valuePrepareString(v ref) (ref, int) + "syscall/js.valuePrepareString": (sp) => { + const str = encoder.encode(String(loadValue(sp + 8))); + storeValue(sp + 16, str); + setInt64(sp + 24, str.length); + }, + + // valueLoadString(v ref, b []byte) + "syscall/js.valueLoadString": (sp) => { + const str = loadValue(sp + 8); + loadSlice(sp + 16).set(str); + }, + + // func valueInstanceOf(v ref, t ref) bool + "syscall/js.valueInstanceOf": (sp) => { + mem().setUint8(sp + 24, loadValue(sp + 8) instanceof loadValue(sp + 16)); + }, + + // func copyBytesToGo(dst []byte, src ref) (int, bool) + "syscall/js.copyBytesToGo": (sp) => { + const dst = loadSlice(sp + 8); + const src = loadValue(sp + 32); + if (!(src instanceof Uint8Array)) { + mem().setUint8(sp + 48, 0); + return; + } + const toCopy = src.subarray(0, dst.length); + dst.set(toCopy); + setInt64(sp + 40, toCopy.length); + mem().setUint8(sp + 48, 1); + }, + + // func copyBytesToJS(dst ref, src []byte) (int, bool) + "syscall/js.copyBytesToJS": (sp) => { + const dst = loadValue(sp + 8); + const src = loadSlice(sp + 16); + if (!(dst instanceof Uint8Array)) { + mem().setUint8(sp + 48, 0); + return; + } + const toCopy = src.subarray(0, dst.length); + dst.set(toCopy); + setInt64(sp + 40, toCopy.length); + mem().setUint8(sp + 48, 1); + }, + + "debug": (value) => { + console.log(value); + }, + } + }; + } + + async run(instance) { + this._inst = instance; + this._values = [ // TODO: garbage collection + NaN, + 0, + null, + true, + false, + global, + this, + ]; + this._refs = new Map(); + this.exited = false; + + const mem = new DataView(this._inst.exports.mem.buffer) + + // Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory. + let offset = 4096; + + const strPtr = (str) => { + const ptr = offset; + const bytes = encoder.encode(str + "\0"); + new Uint8Array(mem.buffer, offset, bytes.length).set(bytes); + offset += bytes.length; + if (offset % 8 !== 0) { + offset += 8 - (offset % 8); + } + return ptr; + }; + + const argc = this.argv.length; + + const argvPtrs = []; + this.argv.forEach((arg) => { + argvPtrs.push(strPtr(arg)); + }); + + const keys = Object.keys(this.env).sort(); + argvPtrs.push(keys.length); + keys.forEach((key) => { + argvPtrs.push(strPtr(`${key}=${this.env[key]}`)); + }); + + const argv = offset; + argvPtrs.forEach((ptr) => { + mem.setUint32(offset, ptr, true); + mem.setUint32(offset + 4, 0, true); + offset += 8; + }); + + this._inst.exports.run(argc, argv); + if (this.exited) { + this._resolveExitPromise(); + } + await this._exitPromise; + } + + _resume() { + if (this.exited) { + throw new Error("Go program has already exited"); + } + this._inst.exports.resume(); + if (this.exited) { + this._resolveExitPromise(); + } + } + + _makeFuncWrapper(id) { + const go = this; + return function () { + const event = { id: id, this: this, args: arguments }; + go._pendingEvent = event; + go._resume(); + return event.result; + }; + } + } + + if ( + global.require && + global.require.main === module && + global.process && + global.process.versions && + !global.process.versions.electron + ) { + if (process.argv.length < 3) { + console.error("usage: go_js_wasm_exec [wasm binary] [arguments]"); + process.exit(1); + } + + const go = new Go(); + go.argv = process.argv.slice(2); + go.env = Object.assign({ TMPDIR: require("os").tmpdir() }, process.env); + go.exit = process.exit; + WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject).then((result) => { + process.on("exit", (code) => { // Node.js exits if no event handler is pending + if (code === 0 && !go.exited) { + // deadlock, make Go print error and stack traces + go._pendingEvent = { id: 0 }; + go._resume(); + } + }); + return go.run(result.instance); + }).catch((err) => { + console.error(err); + process.exit(1); + }); + } +})(); diff --git a/lib.wasm b/lib.wasm new file mode 100755 index 0000000..9067e87 Binary files /dev/null and b/lib.wasm differ diff --git a/masquerade.go b/masquerade.go new file mode 100644 index 0000000..794afc6 --- /dev/null +++ b/masquerade.go @@ -0,0 +1,15 @@ +// +build js,wasm + +package main + +import "github.com/esimov/pigo-wasm-demos/masquerade" + +func main() { + c := masquerade.NewCanvas() + webcam, err := c.StartWebcam() + if err != nil { + c.Alert("Webcam not detected!") + } else { + webcam.Render() + } +} diff --git a/masquerade/canvas.go b/masquerade/canvas.go new file mode 100644 index 0000000..7d1bc5b --- /dev/null +++ b/masquerade/canvas.go @@ -0,0 +1,405 @@ +package masquerade + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "log" + "math" + "net/http" + "net/url" + "syscall/js" + "time" + + "github.com/esimov/pigo-wasm-demos/detector" +) + +// Canvas struct holds the Javascript objects needed for the Canvas creation +type Canvas struct { + done chan struct{} + succCh chan struct{} + errCh chan error + + // DOM elements + window js.Value + doc js.Value + body js.Value + windowSize struct{ width, height int } + + // Canvas properties + canvas js.Value + ctx js.Value + reqID js.Value + renderer js.Func + + // Webcam properties + navigator js.Value + video js.Value + + showPupil bool + showFace bool + showEyeMask bool + showFaceMask bool + showCoord bool + drawCircle bool +} + +type point struct { + x, y int +} + +var ( + images = make([]js.Value, 6) + files = []string{ + "/images/neon-yellow.png", + "/images/sunglasses.png", + "/images/neon-green.png", + "/images/carnival.png", + "/images/carnival2.png", + "/images/neon-disco.png", + } + facemask js.Value + maskImgWidth int + maskImgHeight int + curImgWidth int + curImgHeight int + imgIdx int +) + +var det *detector.Detector + +// NewCanvas creates and initializes the new Canvas element +func NewCanvas() *Canvas { + var c Canvas + c.window = js.Global() + c.doc = c.window.Get("document") + c.body = c.doc.Get("body") + + c.windowSize.width = 1280 + c.windowSize.height = 720 + + c.canvas = c.doc.Call("createElement", "canvas") + c.canvas.Set("width", js.ValueOf(c.windowSize.width)) + c.canvas.Set("height", js.ValueOf(c.windowSize.height)) + c.canvas.Set("id", "canvas") + c.body.Call("appendChild", c.canvas) + + c.ctx = c.canvas.Call("getContext", "2d") + c.showPupil = true + c.showFace = false + c.showEyeMask = true + c.showFaceMask = true + c.drawCircle = false + + det = detector.NewDetector() + return &c +} + +// Render calls the `requestAnimationFrame` Javascript function in asynchronous mode. +func (c *Canvas) Render() { + var data = make([]byte, c.windowSize.width*c.windowSize.height*4) + c.done = make(chan struct{}) + + for i, file := range files { + img := c.loadImage(file) + images[i] = js.Global().Call("eval", "new Image()") + images[i].Set("src", "data:image/png;base64,"+img) + } + if c.showFaceMask { + facemask = js.Global().Call("eval", "new Image()") + facemask.Set("src", "data:image/png;base64,"+c.loadImage("/images/facemask2.png")) + + maskImgWidth = js.ValueOf(facemask.Get("naturalWidth")).Int() + maskImgHeight = js.ValueOf(facemask.Get("naturalHeight")).Int() + } + + curImgWidth = js.ValueOf(images[0].Get("naturalWidth")).Int() + curImgHeight = js.ValueOf(images[0].Get("naturalHeight")).Int() + + if err := det.UnpackCascades(); err == nil { + c.renderer = js.FuncOf(func(this js.Value, args []js.Value) interface{} { + go func() { + c.window.Get("stats").Call("begin") + + width, height := c.windowSize.width, c.windowSize.height + c.reqID = c.window.Call("requestAnimationFrame", c.renderer) + // Draw the webcam frame to the canvas element + c.ctx.Call("drawImage", c.video, 0, 0) + rgba := c.ctx.Call("getImageData", 0, 0, width, height).Get("data") + + // Convert the rgba value of type Uint8ClampedArray to Uint8Array in order to + // be able to transfer it from Javascript to Go via the js.CopyBytesToGo function. + uint8Arr := js.Global().Get("Uint8Array").New(rgba) + js.CopyBytesToGo(data, uint8Arr) + pixels := c.rgbaToGrayscale(data) + res := det.DetectFaces(pixels, height, width) + c.drawDetection(res) + + c.window.Get("stats").Call("end") + }() + return nil + }) + c.window.Call("requestAnimationFrame", c.renderer) + c.detectKeyPress() + <-c.done + } +} + +// Stop stops the rendering. +func (c *Canvas) Stop() { + c.window.Call("cancelAnimationFrame", c.reqID) + c.done <- struct{}{} + close(c.done) +} + +// StartWebcam reads the webcam data and feeds it into the canvas element. +// It returns an empty struct in case of success and error in case of failure. +func (c *Canvas) StartWebcam() (*Canvas, error) { + var err error + c.succCh = make(chan struct{}) + c.errCh = make(chan error) + + c.video = c.doc.Call("createElement", "video") + + // If we don't do this, the stream will not be played. + c.video.Set("autoplay", 1) + c.video.Set("playsinline", 1) // important for iPhones + + // The video should fill out all of the canvas + c.video.Set("width", 0) + c.video.Set("height", 0) + + c.body.Call("appendChild", c.video) + + success := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + go func() { + c.video.Set("srcObject", args[0]) + c.video.Call("play") + c.succCh <- struct{}{} + }() + return nil + }) + + failure := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + go func() { + err = fmt.Errorf("failed initialising the camera: %s", args[0].String()) + c.errCh <- err + }() + return nil + }) + + opts := js.Global().Get("Object").New() + + videoSize := js.Global().Get("Object").New() + videoSize.Set("width", c.windowSize.width) + videoSize.Set("height", c.windowSize.height) + videoSize.Set("aspectRatio", 1.777777778) + + opts.Set("video", videoSize) + opts.Set("audio", false) + + promise := c.window.Get("navigator").Get("mediaDevices").Call("getUserMedia", opts) + promise.Call("then", success, failure) + + select { + case <-c.succCh: + return c, nil + case err := <-c.errCh: + return nil, err + } +} + +// rgbaToGrayscale converts the rgb pixel values to grayscale +func (c *Canvas) rgbaToGrayscale(data []uint8) []uint8 { + rows, cols := c.windowSize.width, c.windowSize.height + for r := 0; r < rows; r++ { + for c := 0; c < cols; c++ { + // gray = 0.2*red + 0.7*green + 0.1*blue + data[r*cols+c] = uint8(math.Round( + 0.2126*float64(data[r*4*cols+4*c+0]) + + 0.7152*float64(data[r*4*cols+4*c+1]) + + 0.0722*float64(data[r*4*cols+4*c+2]))) + } + } + return data +} + +// drawDetection draws the detected faces and eyes. +func (c *Canvas) drawDetection(dets [][]int) { + var p1, p2 point + var imgScale float64 + + for i := 0; i < len(dets); i++ { + if dets[i][3] > 50 { + row, col, scale := dets[i][1], dets[i][0], dets[i][2] + c.ctx.Call("beginPath") + c.ctx.Set("lineWidth", 3) + c.ctx.Set("strokeStyle", "red") + + if c.showFace { + if c.drawCircle { + c.ctx.Call("moveTo", row+int(scale/2), col) + c.ctx.Call("arc", row, col, scale/2, 0, 2*math.Pi, true) + } else { + if c.showCoord { + c.ctx.Set("fillStyle", "red") + c.ctx.Set("font", "18px Arial") + message := fmt.Sprintf("(%v, %v)", row-scale/2, col-scale/2) + txtWidth := c.ctx.Call("measureText", js.ValueOf(message)).Get("width").Int() + c.ctx.Call("fillText", message, (row-scale/2)-txtWidth/2, col-scale/2-10) + } + c.ctx.Call("rect", row-scale/2, col-scale/2, scale, scale) + } + } + c.ctx.Call("stroke") + + if c.showPupil { + leftPupil := det.DetectLeftPupil(dets[i]) + if leftPupil != nil { + if !c.showEyeMask { + col, row, scale := leftPupil.Col, leftPupil.Row, leftPupil.Scale/8 + c.ctx.Call("moveTo", col+int(scale), row) + c.ctx.Call("arc", col, row, scale, 0, 2*math.Pi, true) + } + p1 = point{x: leftPupil.Row, y: leftPupil.Col} + } + + rightPupil := det.DetectRightPupil(dets[i]) + if rightPupil != nil { + if !c.showEyeMask { + col, row, scale := rightPupil.Col, rightPupil.Row, rightPupil.Scale/8 + c.ctx.Call("moveTo", col+int(scale), row) + c.ctx.Call("arc", col, row, scale, 0, 2*math.Pi, true) + } + p2 = point{x: rightPupil.Row, y: rightPupil.Col} + } + c.ctx.Call("stroke") + + if c.showFaceMask && (p1.x != 0 && p2.y != 0) { + points := det.DetectMouthPoints(leftPupil, rightPupil) + p1, p2 := points[0], points[1] + + // Calculate the lean angle between the two mouth points. + angle := 1 - (math.Atan2(float64(p2[0]-p1[0]), float64(p2[1]-p1[1])) * 180 / math.Pi / 90) + if scale < maskImgWidth || scale < maskImgHeight { + if maskImgHeight > maskImgWidth { + imgScale = float64(scale) / float64(maskImgHeight) + } else { + imgScale = float64(scale) / float64(maskImgWidth) + } + } + width, height := float64(maskImgWidth)*imgScale*0.75, float64(maskImgHeight)*imgScale*0.75 + tx := row - int(width/2) + ty := p1[1] + (p1[1]-p2[1])/2 - int(height*0.5) + + c.ctx.Call("save") + c.ctx.Call("translate", js.ValueOf(tx).Int(), js.ValueOf(ty).Int()) + c.ctx.Call("rotate", js.ValueOf(angle).Float()) + c.ctx.Call("drawImage", facemask, + js.ValueOf(0).Int(), js.ValueOf(0).Int(), + js.ValueOf(width).Int(), js.ValueOf(height).Int(), + ) + c.ctx.Call("restore") + } + + if c.showEyeMask && (p1.x != 0 && p2.y != 0) { + // Calculate the lean angle between the pupils. + angle := 1 - (math.Atan2(float64(p2.y-p1.y), float64(p2.x-p1.x)) * 180 / math.Pi / 90) + if scale < curImgWidth || scale < curImgHeight { + if curImgHeight > curImgWidth { + imgScale = float64(scale) / float64(curImgHeight) + } else { + imgScale = float64(scale) / float64(curImgWidth) + } + } + + width, height := float64(curImgWidth)*imgScale, float64(curImgHeight)*imgScale + tx := row - int(width/2) + ty := leftPupil.Row + (leftPupil.Row-rightPupil.Row)/2 - int(height/2) + + c.ctx.Call("save") + c.ctx.Call("translate", js.ValueOf(tx).Int(), js.ValueOf(ty).Int()) + c.ctx.Call("rotate", js.ValueOf(angle).Float()) + c.ctx.Call("drawImage", images[imgIdx], + js.ValueOf(0).Int(), js.ValueOf(0).Int(), + js.ValueOf(width).Int(), js.ValueOf(height).Int(), + ) + c.ctx.Call("restore") + } + } + } + } +} + +// detectKeyPress listen for the keypress event and retrieves the key code. +func (c *Canvas) detectKeyPress() { + keyEventHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + keyCode := args[0].Get("key") + switch { + case keyCode.String() == "q": + c.showFace = !c.showFace + case keyCode.String() == "e": + c.showPupil = !c.showPupil + case keyCode.String() == "a": + c.drawCircle = !c.drawCircle + case keyCode.String() == "r": + c.showEyeMask = !c.showEyeMask + case keyCode.String() == "x": + c.showCoord = !c.showCoord + case keyCode.String() == "f": + c.showFaceMask = !c.showFaceMask + case keyCode.String() == "w": + imgIdx++ + if imgIdx > len(images)-1 { + imgIdx = 0 + } + curImgWidth = js.ValueOf(images[imgIdx].Get("naturalWidth")).Int() + curImgHeight = js.ValueOf(images[imgIdx].Get("naturalHeight")).Int() + case keyCode.String() == "s": + imgIdx-- + if imgIdx < 0 { + imgIdx = len(images) - 1 + } + curImgWidth = js.ValueOf(images[imgIdx].Get("naturalWidth")).Int() + curImgHeight = js.ValueOf(images[imgIdx].Get("naturalHeight")).Int() + default: + c.drawCircle = false + } + return nil + }) + c.doc.Call("addEventListener", "keypress", keyEventHandler) +} + +// Log calls the `console.log` Javascript function +func (c *Canvas) Log(args ...interface{}) { + c.window.Get("console").Call("log", args...) +} + +// Alert calls the `alert` Javascript function +func (c *Canvas) Alert(args ...interface{}) { + alert := c.window.Get("alert") + alert.Invoke(args...) +} + +// loadImage load the source image and encodes it to base64 format. +func (c *Canvas) loadImage(path string) string { + href := js.Global().Get("location").Get("href") + u, err := url.Parse(href.String()) + if err != nil { + log.Fatal(err) + } + u.Path = path + u.RawQuery = fmt.Sprint(time.Now().UnixNano()) + + log.Println("loading image file: " + u.String()) + resp, err := http.Get(u.String()) + if err != nil { + log.Fatal(err) + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatal(err) + } + return base64.StdEncoding.EncodeToString(b) +}