apps/streamana
1
0
Fork 0
mirror of https://github.com/davedoesdev/streamana.git synced 2025-10-17 11:00:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Start restructure to allow video to be restarted

Browse Source
This commit is contained in:
David Halls
2021-08-21 12:08:22 +01:00
parent e863e4443e
commit 02e6e32028
4 changed files with 181 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
site/example.html
View File

@@ -43,11 +43,11 @@
</div>
<div class="form-check">
<input id="lock-portrait" class="form-check-input" type="checkbox">
<label for="lock-portrait" class="form-check-label">Lock and rotate portrait video</label>
<label for="lock-portrait" class="form-check-label">Lock and rotate portrait mode</label>
</div>
<div class="form-check">
<input id="zoom-video" class="form-check-input" type="checkbox">
<label for="zoom-video" class="form-check-label">Display video without vertical bars</label>
<label for="zoom-video" class="form-check-label">Minimize vertical bars in local video display</label>
</div>
<div class="pt-4">
<label for="resolution" class="form-label">Encoder resolution</label>

258
site/example.js
View File

@@ -10,6 +10,8 @@ import {
max_video_encoder_config,
} from './resolution.js';
const target_frame_rate = 30;
const ingestion_url_el = document.getElementById('ingestion-url');
ingestion_url_el.value = localStorage.getItem('streamana-example-ingestion-url');
@@ -144,7 +146,15 @@ async function start() {
error_alert_el_parent.removeChild(error_alert_el);
}
let camera_stream, gl_canvas, canvas_stream, lock_portrait = false, done = false;
// get aspect ratio of encoder
console.log(`encoder resolution: ${video_encoder_config.width}x${video_encoder_config.height}`);
const ar_encoder = video_encoder_config.ratio;
const ar_encoder_inv = 1/ar_encoder;
const zoom_video = zoom_video_el.checked;
const lock_portrait = screen.orientation.type.startsWith('portrait') && lock_portrait_el.checked;
let video_el, gl_canvas, canvas_stream, camera_stream, done = false;
function cleanup(err) {
if (err) {
console.error(err);
@@ -193,130 +203,11 @@ async function start() {
camera_swap_el.classList.add('d-none');
}
try {
// create video element which will be used for grabbing the frames to
// write to a canvas so we can apply webgl shaders
// also used to get the native video dimensions
const video_el = document.createElement('video');
video_el.muted = true;
video_el.playsInline = true;
// Safari on iOS requires us to play() in the click handler and doesn't
// track async calls. So we play a blank video first. After that, the video
// element is blessed for script-driven playback.
video_el.src = 'empty.mp4';
await video_el.play();
// capture video from webcam
const camera_video_constraints = {
width: video_encoder_config.width,
height: video_encoder_config.height,
frameRate: {
ideal: 30,
max: 30
},
facingMode: facing_mode
};
try {
camera_stream = await navigator.mediaDevices.getUserMedia({
audio: true,
video: camera_video_constraints
});
} catch (ex) {
// retry in case audio isn't available
console.warn("Failed to get user media, retrying without audio");
camera_stream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: camera_video_constraints
});
}
const video_settings = camera_stream.getVideoTracks()[0].getSettings();
facing_mode = video_settings.facingMode || 'user';
localStorage.setItem('streamana-facing-mode', facing_mode);
canvas_el.addEventListener('webglcontextlost', cleanup);
// use glsl-canvas to make managing webgl stuff easier
// because it's not visible, client dimensions are zero so we
// need to substitute actual dimensions instead
gl_canvas = new GlCanvas(canvas_el, {
// as an example, greyscale the stream
fragmentString: shader
});
gl_canvas.on('error', cleanup);
// tell canvas to use frames from video
gl_canvas.setTexture('u_texture', video_el);
// wait for video to load (must come after gl_canvas.setTexture() since it
// registers a loadeddata handler which then registers a play handler)
video_el.addEventListener('loadeddata', async function () {
try {
console.log(`video resolution: ${this.videoWidth}x${this.videoHeight}`);
console.log(`encoder resolution: ${video_encoder_config.width}x${video_encoder_config.height}`);
// set aspect ratios of video and encoder
const ar_encoder = video_encoder_config.ratio;
const ar_encoder_inv = 1/ar_encoder;
// set canvas dimensions to same as encoder so its gets all the output
canvas_el.width = video_encoder_config.width;
canvas_el.height = video_encoder_config.height;
// check whether we're locking portrait mode or zooming (display without bars)
let zoom_video = zoom_video_el.checked;
if ((this.videoWidth < this.videoHeight) && lock_portrait_el.checked) {
lock_portrait = true;
// rotate the canvas
canvas_el.classList.add('rotate');
canvas_el.classList.remove('mw-100', 'mh-100');
canvas_el_parent.classList.remove('mx-auto');
// lock to portrait mode
try {
await screen.orientation.lock('portrait');
} catch (ex) {
if (ex.name === 'SecurityError') {
if (!document.fullscreenElement) {
await document.documentElement.requestFullscreen();
}
await screen.orientation.lock('portrait');
} else if (ex.name !== 'NotSupportedError') {
throw ex;
}
}
} else if (zoom_video) {
// we're going to remove the bars for local display only
canvas_el.classList.add('zoom');
canvas_el.classList.remove('mw-100', 'mh-100');
canvas_el_parent.classList.remove('mx-auto');
}
// if we're locked to portrait mode, tell the shader to rotate the video
gl_canvas.setUniform('u_rotate', lock_portrait);
// start the camera video
this.play();
// capture video from the canvas
// Note: Safari on iOS doesn't get any data, might be related to
// https://bugs.webkit.org/show_bug.cgi?id=181663
const frame_rate = video_settings.frameRate;
canvas_stream = canvas_el.captureStream(frame_rate);
// add audio if present
const audio_tracks = camera_stream.getAudioTracks();
if (audio_tracks.length > 0) {
canvas_stream.addTrack(audio_tracks[0]);
}
const update = () => {
function update() {
// update the canvas
if (gl_canvas.onLoop()) {
const ar_video = this.videoWidth / this.videoHeight;
// get aspect ratio of video
const ar_video = video_el.videoWidth / video_el.videoHeight;
// Note: we need to use canvas_el_parent.parentNode.offsetWidth
// to take into account margins
@@ -381,14 +272,82 @@ async function start() {
// chrome bug when rotate (sometimes half page doesn't render)
// chrome inspect not working
// allow select audio and video devices
// will need an audio element for "indirection"
// performance on mobile
// a40 no buffers currently available in the reader queue
// windows, android, iOS, find a mac to test
}
};
}
// start HLS from the canvas stream to the ingestion URL
hls = new HLS(canvas_stream, ingestion_url, ffmpeg_lib_url, frame_rate, lock_portrait);
try {
// create video element which will be used for grabbing the frames to
// write to a canvas so we can apply webgl shaders
// also used to get the native video dimensions
video_el = document.createElement('video');
video_el.muted = true;
video_el.playsInline = true;
// Safari on iOS requires us to play() in the click handler and doesn't
// track async calls. So we play a blank video first. After that, the video
// element is blessed for script-driven playback.
video_el.src = 'empty.mp4';
await video_el.play();
canvas_el.addEventListener('webglcontextlost', cleanup);
// set canvas dimensions to same as encoder so its gets all the output
canvas_el.width = video_encoder_config.width;
canvas_el.height = video_encoder_config.height;
// use glsl-canvas to make managing webgl stuff easier
gl_canvas = new GlCanvas(canvas_el, {
// as an example, greyscale the stream
fragmentString: shader
});
gl_canvas.on('error', cleanup);
// tell canvas to use frames from video
gl_canvas.setTexture('u_texture', video_el);
// check whether we're locking portrait mode or zooming (display without bars)
if (lock_portrait) {
// rotate the canvas
canvas_el.classList.add('rotate');
canvas_el.classList.remove('mw-100', 'mh-100');
canvas_el_parent.classList.remove('mx-auto');
// lock to portrait mode
try {
await screen.orientation.lock('portrait');
} catch (ex) {
if (ex.name === 'SecurityError') {
if (!document.fullscreenElement) {
await document.documentElement.requestFullscreen();
}
await screen.orientation.lock('portrait');
} else if (ex.name !== 'NotSupportedError') {
throw ex;
}
}
} else if (zoom_video) {
// we're going to remove the bars for local display only
canvas_el.classList.add('zoom');
canvas_el.classList.remove('mw-100', 'mh-100');
canvas_el_parent.classList.remove('mx-auto');
}
// if we're locked to portrait mode, tell the shader to rotate the video
gl_canvas.setUniform('u_rotate', lock_portrait);
// capture video from the canvas
// Note: Safari on iOS doesn't get any data, might be related to
// https://bugs.webkit.org/show_bug.cgi?id=181663
//const frame_rate = video_settings.frameRate;
canvas_stream = canvas_el.captureStream(target_frame_rate);
// HLS from the canvas stream to the ingestion URL
hls = new HLS(canvas_stream, ingestion_url, ffmpeg_lib_url, target_frame_rate, lock_portrait);
hls.addEventListener('run', () => console.log('HLS running'));
hls.addEventListener('exit', ev => {
const msg = `HLS exited with status ${ev.detail.code}`;
@@ -405,12 +364,59 @@ async function start() {
this.end(true);
}
waiting_el.classList.add('d-none');
if (!lock_portrait) {
camera_swap_el.classList.remove('d-none');
}
canvas_el.classList.remove('invisible');
go_live_el.disabled = false;
update();
});
hls.addEventListener('update', update);
// capture video from webcam
const camera_video_constraints = {
width: video_encoder_config.width,
height: video_encoder_config.height,
frameRate: {
ideal: target_frame_rate,
max: target_frame_rate
},
facingMode: facing_mode
};
try {
camera_stream = await navigator.mediaDevices.getUserMedia({
audio: true,
video: camera_video_constraints
});
} catch (ex) {
// retry in case audio isn't available
console.warn("Failed to get user media, retrying without audio");
camera_stream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: camera_video_constraints
});
}
const video_settings = camera_stream.getVideoTracks()[0].getSettings();
facing_mode = video_settings.facingMode || 'user';
localStorage.setItem('streamana-facing-mode', facing_mode);
// wait for video to load (must come after gl_canvas.setTexture() since it
// registers a loadeddata handler which then registers a play handler)
video_el.addEventListener('loadeddata', async function () {
try {
console.log(`video resolution: ${this.videoWidth}x${this.videoHeight}`);
// start the camera video
this.play();
// add audio if present
const audio_tracks = camera_stream.getAudioTracks();
if (audio_tracks.length > 0) {
canvas_stream.addTrack(audio_tracks[0]);
}
await hls.start();
} catch (ex) {
cleanup(ex);

2
site/gl-canvas.js
View File

@@ -6,6 +6,8 @@ const { Canvas } = await importUMD('./glsl-canvas.min.js');
export class GlCanvas extends Canvas {
constructor(canvas_el, options) {
super(new Proxy(canvas_el, {
// because it's not visible, client dimensions are zero so we
// need to substitute actual dimensions instead
get: (target, name, receiver) => {
if (name === 'getBoundingClientRect') {
return () => new DOMRect(0, 0, target.width, target.height);

7
site/hls.js
View File

@@ -21,9 +21,14 @@ export class HLS extends EventTarget {
}
this.update_event = new CustomEvent('update');
this.sending = false;
this.started = false;
}
async start() {
if (this.started) {
return;
}
// if audio isn't present, add a silent track
if (this.stream.getAudioTracks().length === 0) {
console.warn("No audio present, adding silence");
@@ -68,6 +73,8 @@ export class HLS extends EventTarget {
console.log("Using MediaRecorder MP4");
}
}
this.started = true;
}
async media_recorder(mimeType) {