diff --git a/ar-filters/app.py b/ar-filters/app.py index c35b378..42bb4e0 100644 --- a/ar-filters/app.py +++ b/ar-filters/app.py @@ -3,17 +3,15 @@ import sys import cv2 import itertools import numpy as np -from time import time import mediapipe as mp -import matplotlib.pyplot as plt import socket import struct mp_face_mesh = mp.solutions.face_mesh face_mesh_videos = mp_face_mesh.FaceMesh(static_image_mode=False, max_num_faces=1, min_detection_confidence=0.5, min_tracking_confidence=0.3) -eye = cv2.imread('/home/kalit/Desktop/GeorgiaTech/Fall_2024/CS_8903/WebRTC_research/ar-filters/filter_imgs/eye.jpg') -mouth = cv2.imread('/home/kalit/Desktop/GeorgiaTech/Fall_2024/CS_8903/WebRTC_research/ar-filters/filter_imgs/smile.png') +eye = cv2.imread('/home/epl/Desktop/WebRTC_research/ar-filters/filter_imgs/eye.jpg') +mouth = cv2.imread('/home/epl/Desktop/WebRTC_research/ar-filters/filter_imgs/smile.png') def detectFacialLandmarks(image, face_mesh): return face_mesh.process(image[:,:,::-1]) @@ -120,9 +118,10 @@ def main(): new_frame = frame _, buffer = cv2.imencode('.jpg', new_frame) - output_file_path = 'output.jpg' - with open(output_file_path, 'wb') as f: - f.write(buffer) + # output_file_path = 'output.jpg' + # with open(output_file_path, 'wb') as f: + # f.write(buffer) + print("Sending back processed image") conn.sendall(struct.pack('!I', len(buffer)) + buffer.tobytes()) conn.close() diff --git a/client/video_stream.go b/client/video_stream.go index 39fb713..67ffda0 100644 --- a/client/video_stream.go +++ b/client/video_stream.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "net" + "os/exec" "time" "github.com/asticode/go-astiav" @@ -35,15 +36,34 @@ func openCameraFeed(peerConnection *webrtc.PeerConnection, videoTrack *webrtc.Tr fmt.Println("Writing to tracks") vp := NewVideoProcessor() - go vp.writeH264ToTrackFFmpegFilters(videoTrack) - + go vp.writeH264ToTrackAR(videoTrack) + // go vp.writeH264ToTrackFFmpegFilters(videoTrack) return nil } +func establishSSHTunnel() (*exec.Cmd, error) { + cmd := exec.Command("ssh", "-L", "5005:127.0.0.1:5005", "-J", "fastvideo", "-i", "~/.ssh/picluster", "epl@10.100.1.165") + err := cmd.Start() + if err != nil { + return nil, fmt.Errorf("failed to start SSH tunnel: %w", err) + } + + fmt.Println("SSH Tunnel established") + return cmd, nil +} + +func closeSSHTunnel(cmd *exec.Cmd) error { + err := cmd.Wait() + if err != nil { + return fmt.Errorf("error waiting for SSH tunnel process: %w", err) + } + return nil +} + func (vp *VideoProcessor) writeH264ToTrackAR(track *webrtc.TrackLocalStaticSample) { defer vp.freeVideoCoding() - conn, err := net.Dial("tcp", "localhost:5005") + conn, err := net.Dial("tcp", "127.0.0.1:5005") if err != nil { panic(err) } @@ -51,6 +71,8 @@ func (vp *VideoProcessor) writeH264ToTrackAR(track *webrtc.TrackLocalStaticSampl ticker := time.NewTicker(h264FrameDuration) for ; true; <-ticker.C { + startTime := time.Now() + if err = vp.inputFormatContext.ReadFrame(vp.decodePacket); err != nil { if errors.Is(err, astiav.ErrEof) { break @@ -66,7 +88,7 @@ func (vp *VideoProcessor) writeH264ToTrackAR(track *webrtc.TrackLocalStaticSampl for { if err = vp.decodeCodecContext.ReceiveFrame(vp.decodeFrame); err != nil { if errors.Is(err, astiav.ErrEof) || errors.Is(err, astiav.ErrEagain) { - fmt.Println("Error while receiving decoded framed: ", err) + // fmt.Println("Error while receiving decoded framed: ", err) break } panic(err) @@ -79,11 +101,13 @@ func (vp *VideoProcessor) writeH264ToTrackAR(track *webrtc.TrackLocalStaticSampl vp.pts++ vp.rgbaFrame.SetPts(vp.pts) - + startTime2 := time.Now() vp.arFilterFrame, err = OverlayARFilter(conn, vp.rgbaFrame) if err != nil { fmt.Println("Failed to add AR filter to frame: ", err) } + elapsedTime2 := time.Since(startTime2) + fmt.Printf("Time taken for adding AR filter: %v\n", elapsedTime2) if err = vp.convertToYUV420PContext.ScaleFrame(vp.arFilterFrame, vp.yuv420PFrame); err != nil { panic(err) @@ -112,6 +136,8 @@ func (vp *VideoProcessor) writeH264ToTrackAR(track *webrtc.TrackLocalStaticSampl } } } + elapsedTime := time.Since(startTime) + fmt.Printf("Time taken from reading the packet, decoding, adding AR filter, encoding to writing in the WebRTC track: %v\n", elapsedTime) } } @@ -123,6 +149,7 @@ func (vp *VideoProcessor) writeH264ToTrackFFmpegFilters(track *webrtc.TrackLocal ticker := time.NewTicker(h264FrameDuration) for ; true; <-ticker.C { + startTime := time.Now() if err = vp.inputFormatContext.ReadFrame(vp.decodePacket); err != nil { if errors.Is(err, astiav.ErrEof) { break @@ -182,5 +209,7 @@ func (vp *VideoProcessor) writeH264ToTrackFFmpegFilters(track *webrtc.TrackLocal } } } + elapsedTime := time.Since(startTime) + fmt.Printf("Time taken from reading the packet, decoding, adding AR filter, encoding to writing in the WebRTC track: %v\n", elapsedTime) } } diff --git a/relevant_repos.txt b/relevant_repos.txt index b1c04dd..d4f85a5 100644 --- a/relevant_repos.txt +++ b/relevant_repos.txt @@ -48,8 +48,21 @@ monitoring tools for battery drains TPUs = Tensor Processing Units Tasks: -1. Start timing on local machine -2. Integrate AR filters with WebRTC video stream -> stuck with yuv422p to rgb conversion -3. WebRTC support for Rasberry Pi. -> jetson -4. Devkit for VR -> hardware test platforms -5. Look for other people working on same problem, google scholar \ No newline at end of file +1. Look for other people working on same problem, google scholar +2. explore AR Workload +3. Volumetric videos - format, can handle with regular RTP? +4. battery savings? + +ffmpeg -i input.webm -f mpegts udp://224.0.0.251:5353 + +/home/kalit/Desktop/GeorgiaTech/Fall_2024/CS_8903/WebRTC_research/ar-filters/filter_imgs/eye.jpg +/home/epl/Desktop/WebRTC_research/ar-filters/filter_imgs/eye.jpg +/home/epl/Desktop/WebRTC_research/ar-filters/filter_imgs/smile.png + + +AR workloads: +https://cuhksz-inml.github.io/full_scene_volumetric_video_dataset/factsfigures.html + + +Forward streams to localhost:5005 to jetson machine's localhost:5005 - +ssh -L 5005:localhost:5005 -J fastvideo -i ~/.ssh/picluster epl@10.100.1.165 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c96270f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +matplotlib==3.5.1 +mediapipe==0.10.11 +opencv-contrib-python==4.9.0.80 +opencv-python==4.9.0.80 \ No newline at end of file