added LPRNet example for license plate detection

README.md

@@ -65,6 +65,8 @@ See the [example](example) directory.
 * Object Detection
   * [YOLOv5 Demo](example/yolov5)  
   * [YOLOv8 Demo](example/yolov8)
+* License Plate Recognition
+  * [LPRNet Demo](example/lprnet) 
 
 
 ## Pooled Runtimes

example/lprnet/README.md

@@ -0,0 +1,61 @@
+# LPRNet Example
+
+## Usage
+
+
+Make sure you have downloaded the data files first for the examples.
+You only need to do this once for all examples.
+
+```
+cd example/
+git clone https://github.com/swdee/go-rknnlite-data.git data
+```
+
+Run the LPRNet example.
+```
+cd example/lprnet
+go run lprnet.go
+```
+
+
+This will result in the output of:
+```
+Driver Version: 0.8.2, API Version: 1.6.0 (9a7b5d24c@2023-12-13T17:31:11)
+Model Input Number: 1, Ouput Number: 1
+Input tensors:
+  index=0, name=input, n_dims=4, dims=[1, 24, 94, 3], n_elems=6768, size=6768, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=0, scale=0.007843
+Output tensors:
+  index=0, name=output, n_dims=3, dims=[1, 68, 18, 0], n_elems=1224, size=1224, fmt=UNDEFINED, type=INT8, qnt_type=AFFINE, zp=47, scale=0.911201
+Model first run speed: inference=7.787585ms, post processing=25.374µs, total time=7.812959ms
+License plate recognition result: 湘F6CL03
+Benchmark time=61.070751ms, count=10, average total time=6.107075ms
+done
+```
+
+To use your own RKNN compiled model and images.
+```
+go run lprnet.go -m <RKNN model file> -i <image file>
+```
+
+## Proprietary Models
+
+This example makes use of the [Chinese License Plate Recognition LPRNet](https://github.com/sirius-ai/LPRNet_Pytorch). 
+You can train your own LPRNet's for other countries but need to initialize
+the `postprocess.NewLPRNet` with your specific `LPRNetParams` containing the
+maximum length of your countries number plates and character set used.
+
+
+## Background
+
+This LPRNet example is a Go conversion of the [C API Example](https://github.com/airockchip/rknn_model_zoo/blob/main/examples/LPRNet/cpp/main.cc)
+
+
+## References
+
+* [LPRNet: License Plate Recognition via Deep Neural Networks](https://arxiv.org/pdf/1806.10447v1) - Original
+paper proposing LPRNet.
+* [An End to End Recognition for License Plates Using Convolutional Neural Networks](https://www.researchgate.net/publication/332650352_An_End_to_End_Recognition_for_License_Plates_Using_Convolutional_Neural_Networks) - A paper
+that looks at LPRNet usage specific to number plates used in China.
+* [Automatic License Plate Recognition](https://hailo.ai/blog/automatic-license-plate-recognition-with-hailo-8/) - An overview
+of creating a full ALPR architecture that uses; Vehicle detection (YOLO),  License Plate Detection (LPDNet), 
+and License Plate Recognition (LPRNet). 

example/lprnet/lprnet.go

@@ -0,0 +1,193 @@
+/*
+Example code showing how to perform inferencing using a LPRnet model
+*/
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/swdee/go-rknnlite"
+	"github.com/swdee/go-rknnlite/postprocess"
+	"gocv.io/x/gocv"
+	"image"
+	"log"
+	"time"
+)
+
+func main() {
+	// disable logging timestamps
+	log.SetFlags(0)
+
+	// read in cli flags
+	modelFile := flag.String("m", "../data/lprnet-rk3588.rknn", "RKNN compiled model file")
+	imgFile := flag.String("i", "../data/lplate.jpg", "Image file to run inference on")
+	flag.Parse()
+
+	// create rknn runtime instance
+	rt, err := rknnlite.NewRuntime(*modelFile, rknnlite.NPUCoreAuto)
+
+	if err != nil {
+		log.Fatal("Error initializing RKNN runtime: ", err)
+	}
+
+	// optional querying of model file tensors and SDK version.  not necessary
+	// for production inference code
+	inputAttrs := optionalQueries(rt)
+
+	// create LPRNet post processor using parameters used during model training
+	lprnetProcesser := postprocess.NewLPRNet(postprocess.LPRNetParams{
+		PlatePositions: 18,
+		PlateChars: []string{
+			"京", "沪", "津", "渝", "冀", "晋", "蒙", "辽", "吉", "黑",
+			"苏", "浙", "皖", "闽", "赣", "鲁", "豫", "鄂", "湘", "粤",
+			"桂", "琼", "川", "贵", "云", "藏", "陕", "甘", "青", "宁",
+			"新",
+			"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+			"A", "B", "C", "D", "E", "F", "G", "H", "J", "K",
+			"L", "M", "N", "P", "Q", "R", "S", "T", "U", "V",
+			"W", "X", "Y", "Z", "I", "O", "-",
+		},
+	})
+
+	// load image
+	img := gocv.IMRead(*imgFile, gocv.IMReadColor)
+
+	if img.Empty() {
+		log.Fatal("Error reading image from: ", *imgFile)
+	}
+
+	// resize image to 94x24
+	cropImg := gocv.NewMat()
+	scaleSize := image.Pt(int(inputAttrs[0].Dims[2]), int(inputAttrs[0].Dims[1]))
+	gocv.Resize(img, &cropImg, scaleSize, 0, 0, gocv.InterpolationArea)
+
+	defer img.Close()
+	defer cropImg.Close()
+
+	start := time.Now()
+
+	// perform inference on image file
+	outputs, err := rt.Inference([]gocv.Mat{cropImg})
+
+	if err != nil {
+		log.Fatal("Runtime inferencing failed with error: ", err)
+	}
+
+	endInference := time.Now()
+
+	// read number plates from outputs
+	plates := lprnetProcesser.ReadPlates(outputs)
+
+	endDetect := time.Now()
+
+	log.Printf("Model first run speed: inference=%s, post processing=%s, total time=%s\n",
+		endInference.Sub(start).String(),
+		endDetect.Sub(endInference).String(),
+		endDetect.Sub(start).String(),
+	)
+
+	for _, plate := range plates {
+		log.Printf("License plate recognition result: %s\n", plate)
+	}
+
+	// free outputs allocated in C memory after you have finished post processing
+	err = outputs.Free()
+
+	if err != nil {
+		log.Fatal("Error freeing Outputs: ", err)
+	}
+
+	// optional code.  run benchmark to get average time of 10 runs
+	runBenchmark(rt, lprnetProcesser, []gocv.Mat{cropImg})
+
+	// close runtime and release resources
+	err = rt.Close()
+
+	if err != nil {
+		log.Fatal("Error closing RKNN runtime: ", err)
+	}
+
+	log.Println("done")
+}
+
+func runBenchmark(rt *rknnlite.Runtime, lprnetProcesser *postprocess.LPRNet,
+	mats []gocv.Mat) {
+
+	count := 10
+	start := time.Now()
+
+	for i := 0; i < count; i++ {
+		// perform inference on image file
+		outputs, err := rt.Inference(mats)
+
+		if err != nil {
+			log.Fatal("Runtime inferencing failed with error: ", err)
+		}
+
+		// post process
+		_ = lprnetProcesser.ReadPlates(outputs)
+
+		err = outputs.Free()
+
+		if err != nil {
+			log.Fatal("Error freeing Outputs: ", err)
+		}
+	}
+
+	end := time.Now()
+	total := end.Sub(start)
+	avg := total / time.Duration(count)
+
+	log.Printf("Benchmark time=%s, count=%d, average total time=%s\n",
+		total.String(), count, avg.String(),
+	)
+}
+
+func optionalQueries(rt *rknnlite.Runtime) []rknnlite.TensorAttr {
+
+	// get SDK version
+	ver, err := rt.SDKVersion()
+
+	if err != nil {
+		log.Fatal("Error initializing RKNN runtime: ", err)
+	}
+
+	fmt.Printf("Driver Version: %s, API Version: %s\n", ver.DriverVersion, ver.APIVersion)
+
+	// get model input and output numbers
+	num, err := rt.QueryModelIONumber()
+
+	if err != nil {
+		log.Fatal("Error querying IO Numbers: ", err)
+	}
+
+	log.Printf("Model Input Number: %d, Ouput Number: %d\n", num.NumberInput, num.NumberOutput)
+
+	// query Input tensors
+	inputAttrs, err := rt.QueryInputTensors()
+
+	if err != nil {
+		log.Fatal("Error querying Input Tensors: ", err)
+	}
+
+	log.Println("Input tensors:")
+
+	for _, attr := range inputAttrs {
+		log.Printf("  %s\n", attr.String())
+	}
+
+	// query Output tensors
+	outputAttrs, err := rt.QueryOutputTensors()
+
+	if err != nil {
+		log.Fatal("Error querying Output Tensors: ", err)
+	}
+
+	log.Println("Output tensors:")
+
+	for _, attr := range outputAttrs {
+		log.Printf("  %s\n", attr.String())
+	}
+
+	return inputAttrs
+}

postprocess/lprnet.go

@@ -0,0 +1,105 @@
+package postprocess
+
+import (
+	"github.com/swdee/go-rknnlite"
+)
+
+// LPRNet defines the struct for LPRNet model inference post processing
+type LPRNet struct {
+	Params LPRNetParams
+}
+
+// LPRNetParams defines the struct containing the LPRNet parameters to use for
+// post processing operations
+type LPRNetParams struct {
+	// PlatePositions is the number of license plate positions to traverse
+	PlatePositions int
+	// PlateChars are the characters on the number plate used to train the model
+	PlateChars []string
+	// numChars is the number of characters in PlateChars
+	numChar int
+}
+
+// NewLPRNet return an instance of the LPRNet post processor
+func NewLPRNet(p LPRNetParams) *LPRNet {
+	l := &LPRNet{
+		Params: p,
+	}
+
+	l.Params.numChar = len(p.PlateChars)
+
+	return l
+}
+
+// ReadPlates takes the RKNN outputs and reads out the license plate numbers
+func (l *LPRNet) ReadPlates(outputs *rknnlite.Outputs) []string {
+
+	results := make([]string, len(outputs.Output))
+
+	for idx, output := range outputs.Output {
+		results[idx] = l.processPlate(output)
+	}
+
+	return results
+}
+
+// processPlate takes a single RKNN Output and returns the number plate as string
+func (l *LPRNet) processPlate(output rknnlite.Output) string {
+
+	// prebs holds the position of the maximum probabilty of matching the
+	// indexed character
+	prebs := make([]int, l.Params.PlatePositions)
+
+	// traverse license plate positions
+	for x := 0; x < l.Params.PlatePositions; x++ {
+		preb := make([]int, l.Params.numChar)
+
+		for y := 0; y < l.Params.numChar; y++ {
+			// get next column
+			val := output.BufFloat[x+y*l.Params.PlatePositions]
+			preb[y] = int(val)
+		}
+
+		prebs[x] = l.argMax(preb)
+	}
+
+	// remove duplicates and blanks
+	noRepeatBlankLabel := []int{}
+	preC := prebs[0]
+
+	if prebs[0] != l.Params.numChar-1 {
+		noRepeatBlankLabel = append(noRepeatBlankLabel, prebs[0])
+	}
+
+	for _, val := range prebs {
+		if val == l.Params.numChar-1 || val == preC {
+			preC = val
+			continue
+		}
+		noRepeatBlankLabel = append(noRepeatBlankLabel, val)
+		preC = val
+	}
+
+	// convert number plate to string
+	plate := ""
+
+	for _, char := range noRepeatBlankLabel {
+		plate += l.Params.PlateChars[char]
+	}
+
+	return plate
+}
+
+// argMax returns the index of the maximum value in the array.
+func (l *LPRNet) argMax(arr []int) int {
+
+	maxIndex := 0
+
+	for i, value := range arr {
+		if value > arr[maxIndex] {
+			maxIndex = i
+		}
+	}
+
+	return maxIndex
+}