Migrate to kbinani/screenshot for screen adapter

https://github.com/kbinani/screenshot is a library to capture desktop
screen. It is cgo free for Windows, Linux, FreeBSD, OpenBSD, NetBSD, and
Solaris.
This commit is contained in:
Lukas Herman
2020-11-21 13:35:46 -08:00
parent 356eee19ce
commit 282d0a4fb4
6 changed files with 91 additions and 392 deletions

4
go.mod
View File

@@ -3,9 +3,13 @@ module github.com/pion/mediadevices
go 1.13
require (
github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd // indirect
github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
github.com/gen2brain/malgo v0.10.24
github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 // indirect
github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f
github.com/lherman-cs/opus v0.0.2
github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55 // indirect
github.com/pion/logging v0.2.2
github.com/pion/rtp v1.6.1
github.com/pion/webrtc/v2 v2.2.26

9
go.sum
View File

@@ -1,3 +1,5 @@
github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd h1:u7K2oMFMd8APDV3fM1j2rO3U/XJf1g1qC3DDTKou8iM=
github.com/BurntSushi/xgb v0.0.0-20201008132610-5f9e7b3c49cd/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539 h1:1aIqYfg9s9RETAJHGfVKZW4ok0b22p4QTwk8MsdRtPs=
github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
@@ -9,6 +11,8 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gen2brain/malgo v0.10.24 h1:q9TFP4lRYpK8UbH3XSa/SNnMwMLUZraRyZt2u+qKYxg=
github.com/gen2brain/malgo v0.10.24/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrUWtAdksnEs=
github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 h1:Y5Q2mEwfzjMt5+3u70Gtw93ZOu2UuPeeeTBDntF7FoY=
github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
@@ -17,6 +21,8 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f h1:5hWo+DzJQSOBl6X+TDac0SPWffRonuRJ2///OYtYRT8=
github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f/go.mod h1:f8GY5V3lRzakvEyr49P7hHRYoHtPr8zvj/7JodCoRzw=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -26,6 +32,8 @@ github.com/lherman-cs/opus v0.0.2 h1:fE9Du3NKXDBztqvoTd6P2y9eJ9vgIHahGK8yQostnhA
github.com/lherman-cs/opus v0.0.2/go.mod h1:v9KQvlDYMuvlwniumBVMlrB0VHQvyTgxNvaXjPmTmps=
github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9 h1:tbuodUh2vuhOVZAdW3NEUvosFHUMJwUNl7jk/VSEiwc=
github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55 h1:4BxFx5XCtXc+nFtXDGDW+Uu5sPtsAbvPh6RObj3fG9o=
github.com/lxn/win v0.0.0-20201111105847-2a20daff6a55/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -111,6 +119,7 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 h1:HlFl4V6pEMziuLXyRkm5BIYq1y1GAbb02pRlWvI54OM=
golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=

View File

@@ -1 +1,79 @@
package screen
import (
"fmt"
"image"
"io"
"github.com/kbinani/screenshot"
"github.com/pion/mediadevices/pkg/driver"
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
type screen struct {
displayIndex int
doneCh chan struct{}
}
func init() {
activeDisplays := screenshot.NumActiveDisplays()
for i := 0; i < activeDisplays; i++ {
priority := driver.PriorityNormal
if i == 0 {
priority = driver.PriorityHigh
}
s := newScreen(i)
driver.GetManager().Register(s, driver.Info{
Label: fmt.Sprint(i),
DeviceType: driver.Screen,
Priority: priority,
})
}
}
func newScreen(displayIndex int) *screen {
s := screen{
displayIndex: displayIndex,
}
return &s
}
func (s *screen) Open() error {
s.doneCh = make(chan struct{})
return nil
}
func (s *screen) Close() error {
close(s.doneCh)
return nil
}
func (s *screen) VideoRecord(selectedProp prop.Media) (video.Reader, error) {
r := video.ReaderFunc(func() (img image.Image, release func(), err error) {
select {
case <-s.doneCh:
return nil, nil, io.EOF
default:
}
img, err = screenshot.CaptureDisplay(s.displayIndex)
release = func() {}
return
})
return r, nil
}
func (s *screen) Properties() []prop.Media {
resolution := screenshot.GetDisplayBounds(s.displayIndex)
supportedProp := prop.Media{
Video: prop.Video{
Width: resolution.Dx(),
Height: resolution.Dy(),
FrameFormat: frame.FormatRGBA,
},
}
return []prop.Media{supportedProp}
}

View File

@@ -1,92 +0,0 @@
package screen
import (
"fmt"
"image"
"time"
"github.com/pion/mediadevices/pkg/driver"
"github.com/pion/mediadevices/pkg/frame"
"github.com/pion/mediadevices/pkg/io/video"
"github.com/pion/mediadevices/pkg/prop"
)
type screen struct {
num int
reader *reader
tick *time.Ticker
}
func deviceID(num int) string {
return fmt.Sprintf("X11Screen%d", num)
}
func init() {
dp, err := openDisplay()
if err != nil {
// No x11 display available.
return
}
defer dp.Close()
numScreen := dp.NumScreen()
for i := 0; i < numScreen; i++ {
driver.GetManager().Register(
&screen{
num: i,
},
driver.Info{
Label: deviceID(i),
DeviceType: driver.Screen,
},
)
}
}
func (s *screen) Open() error {
r, err := newReader(s.num)
if err != nil {
return err
}
s.reader = r
return nil
}
func (s *screen) Close() error {
s.reader.Close()
if s.tick != nil {
s.tick.Stop()
}
return nil
}
func (s *screen) VideoRecord(p prop.Media) (video.Reader, error) {
if p.FrameRate == 0 {
p.FrameRate = 10
}
s.tick = time.NewTicker(time.Duration(float32(time.Second) / p.FrameRate))
var dst image.RGBA
reader := s.reader
r := video.ReaderFunc(func() (image.Image, func(), error) {
<-s.tick.C
return reader.Read().ToRGBA(&dst), func() {}, nil
})
return r, nil
}
func (s *screen) Properties() []prop.Media {
rect := s.reader.img.Bounds()
w := rect.Dx()
h := rect.Dy()
return []prop.Media{
{
DeviceID: deviceID(s.num),
Video: prop.Video{
Width: w,
Height: h,
FrameFormat: frame.FormatRGBA,
},
},
}
}

View File

@@ -1,283 +0,0 @@
package screen
// #cgo pkg-config: x11 xext
// #include <stdint.h>
// #include <sys/shm.h>
// #include <X11/Xlib.h>
// #define XUTIL_DEFINE_FUNCTIONS
// #include <X11/Xutil.h>
// #include <X11/extensions/XShm.h>
//
// void copyBGR24(void *dst, char *src, size_t l) { // 64bit aligned copy
// uint64_t *d = (uint64_t*)dst;
// uint64_t *s = (uint64_t*)src;
// l /= 8;
// for (size_t i = 0; i < l; i ++) {
// uint64_t v = *s;
// // Reorder BGR to RGB
// *d = 0xFF000000FF000000 |
// ((v >> 16) & 0xFF00000000) | (v & 0xFF0000000000) | ((v & 0xFF00000000) << 16) |
// ((v >> 16) & 0xFF) | (v & 0xFF00) | ((v & 0xFF) << 16);
// d++;
// s++;
// }
// }
//
// void copyBGR16(void *dst, char *src, size_t l) { // 64bit aligned copy
// uint64_t *d = (uint64_t*)dst;
// uint32_t *s = (uint32_t*)src;
// l /= 8;
// for (size_t i = 0; i < l; i ++) {
// uint64_t v = *s;
// // Reorder BGR to RGB
// *d = 0xFF000000FF000000 |
// ((v & 0xF8000000) << 8) | ((v & 0x7E00000) << 21) | ((v & 0x1F0000) << 35) |
// ((v & 0xF800) >> 8) | ((v & 0x7E0) << 5) | ((v & 0x1F) << 19);
// d++;
// s++;
// }
// }
//
// char *align64(char *ptr) { // return 64bit aligned pointer
// if (((size_t)ptr & 0x07) == 0) {
// return ptr;
// }
// // Clear lower 3bits to align the address to 8bytes.
// return (char*)(((size_t)ptr & (~(size_t)0x07)) + 0x08);
// }
// size_t align64ForTest(size_t ptr) {
// return (size_t)align64((char*)ptr);
// }
import "C"
import (
"errors"
"fmt"
"image"
"image/color"
"unsafe"
)
const shmaddrInvalid = ^uintptr(0)
type display C.Display
type pixelFormat int
const (
pixFmtBGR24 pixelFormat = iota
pixFmtRGB24
pixFmtBGR16
pixFmtRGB16
)
func openDisplay() (*display, error) {
dp := C.XOpenDisplay(nil)
if dp == nil {
return nil, errors.New("failed to open display")
}
return (*display)(dp), nil
}
func (d *display) c() *C.Display {
return (*C.Display)(d)
}
func (d *display) Close() {
C.XCloseDisplay(d.c())
}
func (d *display) NumScreen() int {
return int(C.XScreenCount(d.c()))
}
type shmImage struct {
dp *C.Display
img *C.XImage
shm C.XShmSegmentInfo
b []byte
pixFmt pixelFormat
}
func (s *shmImage) Free() {
if s.img != nil {
C.XShmDetach(s.dp, &s.shm)
C.XDestroyImage(s.img)
}
if uintptr(unsafe.Pointer(s.shm.shmaddr)) != shmaddrInvalid {
C.shmdt(unsafe.Pointer(s.shm.shmaddr))
}
}
func (s *shmImage) ColorModel() color.Model {
return color.RGBAModel
}
func (s *shmImage) Bounds() image.Rectangle {
return image.Rect(0, 0, int(s.img.width), int(s.img.height))
}
type colorFunc func() (r, g, b, a uint32)
func (c colorFunc) RGBA() (r, g, b, a uint32) {
return c()
}
func (s *shmImage) At(x, y int) color.Color {
switch s.pixFmt {
case pixFmtBGR24:
addr := (x + y*int(s.img.width)) * 4
b := uint32(s.b[addr]) * 0x100
g := uint32(s.b[addr+1]) * 0x100
r := uint32(s.b[addr+2]) * 0x100
return colorFunc(func() (_, _, _, _ uint32) {
return r, g, b, 0xFFFF
})
case pixFmtBGR16:
addr := (x + y*int(s.img.width)) * 2
b1, b2 := s.b[addr], s.b[addr+1]
b := uint32(b1>>3) * 0x100
g := uint32((b1&0x7)<<3|(b2&0xE0)>>5) * 0x100
r := uint32(b2&0x1F) * 0x100
return colorFunc(func() (_, _, _, _ uint32) {
return r, g, b, 0xFFFF
})
default:
panic("unsupported pixel format")
}
}
func (s *shmImage) RGBAAt(x, y int) color.RGBA {
switch s.pixFmt {
case pixFmtBGR24:
addr := (x + y*int(s.img.width)) * 4
b := s.b[addr]
g := s.b[addr+1]
r := s.b[addr+2]
return color.RGBA{R: r, G: g, B: b, A: 0xFF}
case pixFmtBGR16:
addr := (x + y*int(s.img.width)) * 2
b1, b2 := s.b[addr], s.b[addr+1]
b := b1 >> 3
g := (b1&0x7)<<3 | (b2&0xE0)>>5
r := b2 & 0x1F
return color.RGBA{R: r, G: g, B: b, A: 0xFF}
default:
panic("unsupported pixel format")
}
}
func (s *shmImage) ToRGBA(dst *image.RGBA) *image.RGBA {
dst.Rect = s.Bounds()
dst.Stride = int(s.img.width) * 4
l := int(4 * s.img.width * s.img.height)
if len(dst.Pix) < l {
if cap(dst.Pix) < l {
dst.Pix = make([]uint8, l)
}
dst.Pix = dst.Pix[:l]
}
switch s.pixFmt {
case pixFmtBGR24:
C.copyBGR24(unsafe.Pointer(&dst.Pix[0]), s.img.data, C.ulong(len(dst.Pix)))
return dst
case pixFmtBGR16:
C.copyBGR16(unsafe.Pointer(&dst.Pix[0]), s.img.data, C.ulong(len(dst.Pix)))
return dst
default:
panic("unsupported pixel format")
}
}
func newShmImage(dp *C.Display, screen int) (*shmImage, error) {
cScreen := C.int(screen)
w := int(C.XDisplayWidth(dp, cScreen))
h := int(C.XDisplayHeight(dp, cScreen))
v := C.XDefaultVisual(dp, cScreen)
depth := int(C.XDefaultDepth(dp, cScreen))
s := &shmImage{dp: dp}
switch {
case v.red_mask == 0xFF0000 && v.green_mask == 0xFF00 && v.blue_mask == 0xFF:
s.pixFmt = pixFmtBGR24
case v.red_mask == 0xF800 && v.green_mask == 0x7E0 && v.blue_mask == 0x1F:
s.pixFmt = pixFmtBGR16
default:
fmt.Printf("x11capture: unsupported pixel format (R: %0x, G: %0x, B: %0x)\n",
v.red_mask, v.green_mask, v.blue_mask)
return nil, errors.New("unsupported pixel format")
}
s.shm.shmid = C.shmget(C.IPC_PRIVATE, C.ulong(w*h*4+8), C.IPC_CREAT|0600)
if s.shm.shmid == -1 {
return nil, errors.New("failed to get shared memory")
}
s.shm.shmaddr = (*C.char)(C.shmat(s.shm.shmid, unsafe.Pointer(nil), 0))
if uintptr(unsafe.Pointer(s.shm.shmaddr)) == shmaddrInvalid {
s.shm.shmaddr = nil
return nil, errors.New("failed to get shared memory address")
}
s.shm.readOnly = 0
C.shmctl(s.shm.shmid, C.IPC_RMID, nil)
s.img = C.XShmCreateImage(
dp, v, C.uint(depth), C.ZPixmap, C.align64(s.shm.shmaddr), &s.shm, C.uint(w), C.uint(h))
if s.img == nil {
s.Free()
return nil, errors.New("failed to create XShm image")
}
C.XShmAttach(dp, &s.shm)
C.XSync(dp, 0)
return s, nil
}
type reader struct {
dp *C.Display
img *shmImage
}
func newReader(screen int) (*reader, error) {
dp := C.XOpenDisplay(nil)
if dp == nil {
return nil, errors.New("failed to open display")
}
if C.XShmQueryExtension(dp) == 0 {
return nil, errors.New("no XShm support")
}
img, err := newShmImage(dp, screen)
if err != nil {
C.XCloseDisplay(dp)
return nil, err
}
return &reader{
dp: dp,
img: img,
}, nil
}
func (r *reader) Size() (int, int) {
return int(r.img.img.width), int(r.img.img.height)
}
func (r *reader) Read() *shmImage {
C.XShmGetImage(r.dp, C.XDefaultRootWindow(r.dp), r.img.img, 0, 0, C.AllPlanes)
r.img.b = C.GoBytes(
unsafe.Pointer(r.img.img.data),
C.int(r.img.img.width*r.img.img.height*4),
)
return r.img
}
func (r *reader) Close() {
r.img.Free()
C.XCloseDisplay(r.dp)
}
// cAlign64 is fot testing
func cAlign64(ptr uintptr) uintptr {
return uintptr(C.align64ForTest(C.ulong(uintptr(ptr))))
}

View File

@@ -1,17 +0,0 @@
package screen
import (
"testing"
)
func TestAlign64(t *testing.T) {
if ret := cAlign64(0x00010008); ret != 0x00010008 {
t.Errorf("Wrong alignment, expected %x, got %x", 0x00010008, ret)
}
if ret := cAlign64(0x00010006); ret != 0x00010008 {
t.Errorf("Wrong alignment, expected %x, got %x", 0x00010008, ret)
}
if ret := cAlign64(0x00010009); ret != 0x00010010 {
t.Errorf("Wrong alignment, expected %x, got %x", 0x00010010, ret)
}
}