caire/carver.go

package caire

import (
	"fmt"
	"image"
	"image/color"
	"image/draw"
	"math"

	"github.com/esimov/caire/utils"
	pigo "github.com/esimov/pigo/core"
)

// SeamCarver defines the Carve interface method, which have to be
// implemented by the Processor struct.
type SeamCarver interface {
	Resize(*image.NRGBA) (image.Image, error)
}

// maxFaceDetAttempts defines the maximum number of attempts of face detections
const maxFaceDetAttempts = 20

var (
	detAttempts    int
	isFaceDetected bool
)

var (
	sobel       *image.NRGBA
	energySeams = make([][]Seam, 0)
)

// Carver is the main entry struct having as parameters the newly generated image width, height and seam points.
type Carver struct {
	Points []float64
	Seams  []Seam
	Width  int
	Height int
}

// Seam struct contains the seam pixel coordinates.
type Seam struct {
	X int
	Y int
}

// NewCarver returns an initialized Carver structure.
func NewCarver(width, height int) *Carver {
	return &Carver{
		Points: make([]float64, width*height),
		Seams:  []Seam{},
		Width:  width,
		Height: height,
	}
}

// Get energy pixel value.
func (c *Carver) get(x, y int) float64 {
	px := x + y*c.Width
	return c.Points[px]
}

// Set energy pixel value.
func (c *Carver) set(x, y int, px float64) {
	idx := x + y*c.Width
	c.Points[idx] = px
}

// ComputeSeams compute the minimum energy level based on the following logic:
//
//   - traverse the image from the second row to the last row
//     and compute the cumulative minimum energy M for all possible
//     connected seams for each entry (i, j).
//
//   - the minimum energy level is calculated by summing up the current pixel value
//     with the minimum pixel value of the neighboring pixels from the previous row.
func (c *Carver) ComputeSeams(p *Processor, img *image.NRGBA) (*image.NRGBA, error) {
	width, height := img.Bounds().Dx(), img.Bounds().Dy()
	sobel = c.SobelDetector(img, float64(p.SobelThreshold))

	dets := []pigo.Detection{}

	if p.FaceDetector != nil && p.FaceDetect && detAttempts < maxFaceDetAttempts {
		var ratio float64

		if width < height {
			ratio = float64(width) / float64(height)
		} else {
			ratio = float64(height) / float64(width)
		}
		minSize := float64(utils.Min(width, height)) * ratio / 3

		// Transform the image to pixel array.
		pixels := rgbToGrayscale(img)

		cParams := pigo.CascadeParams{
			MinSize:     int(minSize),
			MaxSize:     utils.Min(width, height),
			ShiftFactor: 0.1,
			ScaleFactor: 1.1,

			ImageParams: pigo.ImageParams{
				Pixels: pixels,
				Rows:   height,
				Cols:   width,
				Dim:    width,
			},
		}
		if p.vRes {
			p.FaceAngle = 0.2
		}
		// Run the classifier over the obtained leaf nodes and return the detection results.
		// The result contains quadruplets representing the row, column, scale and detection score.
		dets = p.FaceDetector.RunCascade(cParams, p.FaceAngle)

		// Calculate the intersection over union (IoU) of two clusters.
		dets = p.FaceDetector.ClusterDetections(dets, 0.1)

		if len(dets) == 0 {
			// Retry detecting faces for a certain amount of time.
			if detAttempts < maxFaceDetAttempts {
				detAttempts++
			}
		} else {
			detAttempts = 0
			isFaceDetected = true
		}
	}

	// Traverse the pixel data of the binary file used for protecting the regions
	// which we do not want to be altered by the seam carver,
	// obtain the white patches and apply it to the sobel image.
	if len(p.MaskPath) > 0 && p.Mask != nil {
		p.DebugMask = image.NewNRGBA(img.Bounds())

		for i := 0; i < width*height; i++ {
			x := i % width
			y := (i - x) / width

			r, g, b, _ := p.Mask.At(x, y).RGBA()
			if r>>8 == 0xff && g>>8 == 0xff && b>>8 == 0xff {
				if isFaceDetected {
					// Reduce the brightness of the mask with a small factor if human faces are detected.
					// This way we can avoid the seam carver to remove
					// the pixels inside the detected human faces.
					sobel.Set(x, y, color.RGBA{R: 225, G: 225, B: 225, A: 255})
				} else {
					sobel.Set(x, y, color.White)
				}
			}
		}
	}

	// Traverse the pixel data of the binary file used to remove the image regions
	// we do not want to be retained in the final image, obtain the white patches,
	// but this time inverse the colors to black and merge it back to the sobel image.
	if len(p.RMaskPath) > 0 && p.RMask != nil {
		p.DebugMask = image.NewNRGBA(img.Bounds())

		for i := 0; i < width*height; i++ {
			x := i % width
			y := (i - x) / width

			r, g, b, _ := p.RMask.At(x, y).RGBA()
			// Replace the white pixels with black.
			if r>>8 == 0xff && g>>8 == 0xff && b>>8 == 0xff {
				if isFaceDetected {
					// Reduce the brightness of the mask with a small factor if human faces are detected.
					// This way we can avoid the seam carver to remove
					// the pixels inside the detected human faces.
					sobel.Set(x, y, color.RGBA{R: 25, G: 25, B: 25, A: 255})
				} else {
					sobel.Set(x, y, color.Black)
				}
				p.DebugMask.Set(x, y, color.Black)
			} else {
				p.DebugMask.Set(x, y, color.Transparent)
			}
		}
	}

	// Iterate over the detected faces and fill out the rectangles with white.
	// We need to trick the sobel detector to consider them as important image parts.
	for _, face := range dets {
		if (p.NewHeight != 0 && p.NewHeight < face.Scale) ||
			(p.NewWidth != 0 && p.NewWidth < face.Scale) {
			return nil, fmt.Errorf("%s %s",
				"cannot resize the image to the specified dimension without face deformation.\n",
				"\tRemove the face detection option in case you still wish to resize the image.")
		}
		if face.Q > 5.0 {
			scale := int(float64(face.Scale) / 1.7)
			rect := image.Rect(
				face.Col-scale,
				face.Row-scale,
				face.Col+scale,
				face.Row+scale,
			)
			p.DebugMask = image.NewNRGBA(img.Bounds())
			draw.Draw(sobel, rect, &image.Uniform{color.White}, image.Point{}, draw.Src)
			draw.Draw(p.DebugMask, rect, &image.Uniform{color.White}, image.Point{}, draw.Src)
		}
	}

	// Increase the energy value for each of the selected seam from the seams table
	// in order to avoid picking the same seam over and over again.
	// We expand the energy level of the selected seams to have a better redistribution.
	if len(energySeams) > 0 {
		for i := 0; i < len(energySeams); i++ {
			for _, seam := range energySeams[i] {
				sobel.Set(seam.X, seam.Y, &image.Uniform{color.White})
			}
		}
	}

	var srcImg *image.NRGBA
	if p.BlurRadius > 0 {
		srcImg = image.NewNRGBA(img.Bounds())
		err := Stackblur(srcImg, sobel, uint32(p.BlurRadius))
		if err != nil {
			return nil, fmt.Errorf("error bluring the image: %w", err)
		}
	} else {
		srcImg = sobel
	}

	for x := 0; x < c.Width; x++ {
		for y := 0; y < c.Height; y++ {
			r, _, _, a := srcImg.At(x, y).RGBA()
			c.set(x, y, float64(r)/float64(a))
		}
	}

	var left, middle, right float64

	// Traverse the image from top to bottom and compute the minimum energy level.
	// For each pixel in a row we compute the energy of the current pixel
	// plus the energy of one of the three possible pixels above it.
	for y := 1; y < c.Height; y++ {
		for x := 1; x < c.Width-1; x++ {
			left = c.get(x-1, y-1)
			middle = c.get(x, y-1)
			right = c.get(x+1, y-1)
			min := math.Min(math.Min(left, middle), right)
			// Set the minimum energy level.
			c.set(x, y, c.get(x, y)+min)
		}
		// Special cases: pixels are far left or far right
		left := c.get(0, y) + math.Min(c.get(0, y-1), c.get(1, y-1))
		c.set(0, y, left)
		right := c.get(0, y) + math.Min(c.get(c.Width-1, y-1), c.get(c.Width-2, y-1))
		c.set(c.Width-1, y, right)
	}
	return srcImg, nil
}

// FindLowestEnergySeams find the lowest vertical energy seam.
func (c *Carver) FindLowestEnergySeams(p *Processor) []Seam {
	// Find the lowest cost seam from the energy matrix starting from the last row.
	var (
		min = math.MaxFloat64
		px  int
	)
	seams := make([]Seam, 0)

	// Find the pixel on the last row with the minimum cumulative energy and use this as the starting pixel
	for x := 0; x < c.Width; x++ {
		seam := c.get(x, c.Height-1)
		if seam < min {
			min = seam
			px = x
		}
	}

	seams = append(seams, Seam{X: px, Y: c.Height - 1})
	var left, middle, right float64

	// Walk up in the matrix table, check the immediate three top pixels seam level
	// and add that one which has the lowest cumulative energy.
	for y := c.Height - 2; y >= 0; y-- {
		middle = c.get(px, y)
		// Leftmost seam, no child to the left
		if px == 0 {
			right = c.get(px+1, y)
			if right < middle {
				px++
			}
			// Rightmost seam, no child to the right
		} else if px == c.Width-1 {
			left = c.get(px-1, y)
			if left < middle {
				px--
			}
		} else {
			left = c.get(px-1, y)
			right = c.get(px+1, y)
			min := math.Min(math.Min(left, middle), right)

			if min == left {
				px--
			} else if min == right {
				px++
			}
		}
		seams = append(seams, Seam{X: px, Y: y})
	}

	// compare against c.Width and NOT c.Height, because the image is rotated.
	if p.NewWidth > c.Width || (p.NewHeight > 0 && p.NewHeight > c.Width) {
		// Include the currently processed energy seam into the seams table,
		// but only when an image enlargement operation is commenced.
		// We need to take this approach in order to avoid picking the same seam each time.
		energySeams = append(energySeams, seams)
	}
	return seams
}

// RemoveSeam remove the least important columns based on the stored energy (seams) level.
func (c *Carver) RemoveSeam(img *image.NRGBA, seams []Seam, debug bool) *image.NRGBA {
	bounds := img.Bounds()
	// Reduce the image width with one pixel on each iteration.
	dst := image.NewNRGBA(image.Rect(0, 0, bounds.Dx()-1, bounds.Dy()))

	for _, seam := range seams {
		y := seam.Y
		for x := 0; x < bounds.Max.X; x++ {
			if seam.X == x {
				if debug {
					c.Seams = append(c.Seams, Seam{X: x, Y: y})
				}
			} else if seam.X < x {
				dst.Set(x-1, y, img.At(x, y))
			} else {
				dst.Set(x, y, img.At(x, y))
			}
		}
	}
	return dst
}

// AddSeam add a new seam.
func (c *Carver) AddSeam(img *image.NRGBA, seams []Seam, debug bool) *image.NRGBA {
	var (
		lr, lg, lb uint32
		rr, rg, rb uint32
	)

	bounds := img.Bounds()
	dst := image.NewNRGBA(image.Rect(0, 0, bounds.Dx()+1, bounds.Dy()))

	for _, seam := range seams {
		y := seam.Y
		for x := 0; x < bounds.Max.X; x++ {
			if seam.X == x {
				if debug {
					c.Seams = append(c.Seams, Seam{X: x, Y: y})
				}
				if x > 0 && x != bounds.Max.X {
					lr, lg, lb, _ = img.At(x-1, y).RGBA()
				} else {
					lr, lg, lb, _ = img.At(x, y).RGBA()
				}

				if x < bounds.Max.X-1 {
					rr, rg, rb, _ = img.At(x+1, y).RGBA()
				} else if x == bounds.Max.X {
					rr, rg, rb, _ = img.At(x, y).RGBA()
				}

				// calculate the average color of the neighboring pixels
				avr, avg, avb := (lr+rr)>>1, (lg+rg)>>1, (lb+rb)>>1
				dst.Set(x, y, color.RGBA{uint8(avr >> 8), uint8(avg >> 8), uint8(avb >> 8), 0xff})
				dst.Set(x+1, y, img.At(x, y))
			} else if seam.X < x {
				dst.Set(x, y, img.At(x-1, y))
				dst.Set(x+1, y, img.At(x, y))
			} else {
				dst.Set(x, y, img.At(x, y))
			}
		}
	}

	return dst
}