apps/streamana
1
0
Fork 0
mirror of https://github.com/davedoesdev/streamana.git synced 2025-10-30 08:16:20 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix audio but no video

Browse Source
This commit is contained in:
David Halls
2021-09-01 22:25:34 +01:00
parent dc1631fbac
commit cdcdd26737
4 changed files with 159 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
site/example-shader.js
View File

@@ -5,10 +5,11 @@ uniform sampler2D u_texture;
uniform vec2 u_resolution;
uniform bool u_rotate;
uniform bool u_greyscale;
uniform bool u_active;
out vec4 colour;
vec4 grey(float x, float y) {
vec4 tpix(float x, float y) {
vec3 color = texture(u_texture, vec2(x, y)).rgb;
if (!u_greyscale) {
return vec4(color, 1.0);
@@ -18,6 +19,10 @@ vec4 grey(float x, float y) {
}
void main() {
if (!u_active) {
colour = vec4(0, 0, 0, 1.0);
return;
}
ivec2 size = textureSize(u_texture, 0);
float ar_texture = float(size.x) / float(size.y);
if (u_rotate) {
@@ -29,7 +34,7 @@ void main() {
(gl_FragCoord.y >= (border_height + height))) {
colour = vec4(0);
} else {
colour = grey(1.0 - (gl_FragCoord.y - border_height) / height,
colour = tpix(1.0 - (gl_FragCoord.y - border_height) / height,
gl_FragCoord.x / u_resolution.x);
}
} else {
@@ -39,7 +44,7 @@ void main() {
(gl_FragCoord.x >= (border_width + width))) {
colour = vec4(0);
} else {
colour = grey(1.0 - gl_FragCoord.y / u_resolution.y,
colour = tpix(1.0 - gl_FragCoord.y / u_resolution.y,
(gl_FragCoord.x - border_width) / width);
}
}
@@ -52,7 +57,7 @@ void main() {
(gl_FragCoord.x >= (border_width + width))) {
colour = vec4(0);
} else {
colour = grey((gl_FragCoord.x - border_width) / width,
colour = tpix((gl_FragCoord.x - border_width) / width,
gl_FragCoord.y / u_resolution.y);
}
} else {
@@ -62,7 +67,7 @@ void main() {
(gl_FragCoord.y >= (border_height + height))) {
colour = vec4(0);
} else {
colour = grey(gl_FragCoord.x / u_resolution.x,
colour = tpix(gl_FragCoord.x / u_resolution.x,
(gl_FragCoord.y - border_height) / height);
}
}

16
site/example.css
View File

@@ -16,3 +16,19 @@ canvas.zoom {
.camera-swap-icon {
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Cpath d='M11.534 7h3.932a.25.25 0 0 1 .192.41l-1.966 2.36a.25.25 0 0 1-.384 0l-1.966-2.36a.25.25 0 0 1 .192-.41zm-11 2h3.932a.25.25 0 0 0 .192-.41L2.692 6.23a.25.25 0 0 0-.384 0L.342 8.59A.25.25 0 0 0 .534 9z'/%3E%3Cpath fill-rule='evenodd' d='M8 3c-1.552 0-2.94.707-3.857 1.818a.5.5 0 1 1-.771-.636A6.002 6.002 0 0 1 13.917 7H12.9A5.002 5.002 0 0 0 8 3zM3.1 9a5.002 5.002 0 0 0 8.757 2.182.5.5 0 1 1 .771.636A6.002 6.002 0 0 1 2.083 9H3.1z'/%3E%3C/svg%3E") !important;
}
.mic-icon {
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Cpath d='M3.5 6.5A.5.5 0 0 1 4 7v1a4 4 0 0 0 8 0V7a.5.5 0 0 1 1 0v1a5 5 0 0 1-4.5 4.975V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 .5-.5z'/%3E%3Cpath d='M10 8a2 2 0 1 1-4 0V3a2 2 0 1 1 4 0v5zM8 0a3 3 0 0 0-3 3v5a3 3 0 0 0 6 0V3a3 3 0 0 0-3-3z'/%3E%3C/svg%3E") !important;
}
.mic-icon.off {
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Cpath d='M13 8c0 .564-.094 1.107-.266 1.613l-.814-.814A4.02 4.02 0 0 0 12 8V7a.5.5 0 0 1 1 0v1zm-5 4c.818 0 1.578-.245 2.212-.667l.718.719a4.973 4.973 0 0 1-2.43.923V15h3a.5.5 0 0 1 0 1h-7a.5.5 0 0 1 0-1h3v-2.025A5 5 0 0 1 3 8V7a.5.5 0 0 1 1 0v1a4 4 0 0 0 4 4zm3-9v4.879l-1-1V3a2 2 0 0 0-3.997-.118l-.845-.845A3.001 3.001 0 0 1 11 3z'/%3E%3Cpath d='m9.486 10.607-.748-.748A2 2 0 0 1 6 8v-.878l-1-1V8a3 3 0 0 0 4.486 2.607zm-7.84-9.253 12 12 .708-.708-12-12-.708.708z'/%3E%3C/svg%3E") !important;
}
.camera-icon {
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Cpath fill-rule='evenodd' d='M0 5a2 2 0 0 1 2-2h7.5a2 2 0 0 1 1.983 1.738l3.11-1.382A1 1 0 0 1 16 4.269v7.462a1 1 0 0 1-1.406.913l-3.111-1.382A2 2 0 0 1 9.5 13H2a2 2 0 0 1-2-2V5zm11.5 5.175 3.5 1.556V4.269l-3.5 1.556v4.35zM2 4a1 1 0 0 0-1 1v6a1 1 0 0 0 1 1h7.5a1 1 0 0 0 1-1V5a1 1 0 0 0-1-1H2z'/%3E%3C/svg%3E") !important;
}
.camera-icon.off {
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Cpath fill-rule='evenodd' d='M10.961 12.365a1.99 1.99 0 0 0 .522-1.103l3.11 1.382A1 1 0 0 0 16 11.731V4.269a1 1 0 0 0-1.406-.913l-3.111 1.382A2 2 0 0 0 9.5 3H4.272l.714 1H9.5a1 1 0 0 1 1 1v6a1 1 0 0 1-.144.518l.605.847zM1.428 4.18A.999.999 0 0 0 1 5v6a1 1 0 0 0 1 1h5.014l.714 1H2a2 2 0 0 1-2-2V5c0-.675.334-1.272.847-1.634l.58.814zM15 11.73l-3.5-1.555v-4.35L15 4.269v7.462zm-4.407 3.56-10-14 .814-.58 10 14-.814.58z'/%3E%3C/svg%3E") !important;
}

10
site/example.html
View File

@@ -34,6 +34,16 @@
<span class="navbar-toggler-icon camera-swap-icon"></span>
</button>
</div>
<div id="mic" class="col-sm col-auto order-1">
<button type="button" class="navbar-toggler">
<span id="mic-icon" class="navbar-toggler-icon mic-icon off"></span>
</button>
</div>
<div id="camera" class="col-sm col-auto order-1">
<button type="button" class="navbar-toggler">
<span id="camera-icon" class="navbar-toggler-icon camera-icon off"></span>
</button>
</div>
</div>
</div>
</nav>

157
site/example.js
View File

@@ -84,6 +84,34 @@ document.body.addEventListener('click', function (ev) {
}
});
const mic_el = document.getElementById('mic');
const mic_icon_el = document.getElementById('mic-icon');
if (!!localStorage.getItem('streamana-mic-on')) {
mic_icon_el.classList.remove('off');
}
mic_el.addEventListener('click', function () {
mic_icon_el.classList.toggle('off');
});
function mic_save() {
localStorage.setItem('streamana-mic-on', mic_icon_el.classList.contains('off') ? '' : 'true');
}
mic_el.addEventListener('click', mic_save);
const camera_el = document.getElementById('camera');
const camera_icon_el = document.getElementById('camera-icon');
if (!!localStorage.getItem('streamana-camera-on')) {
camera_icon_el.classList.remove('off');
}
camera_el.addEventListener('click', function () {
camera_icon_el.classList.toggle('off');
});
function camera_save() {
localStorage.setItem('streamana-camera-on', camera_icon_el.classList.contains('off') ? '' : 'true');
}
camera_el.addEventListener('click', camera_save);
const camera_swap_el = document.getElementById('camera-swap');
let video_encoder_config;
let preferred_resolution = localStorage.getItem('streamana-resolution');
if (preferred_resolution) {
@@ -123,8 +151,6 @@ resolution_el.addEventListener('change', function (ev) {
}));
});
const camera_swap_el = document.getElementById('camera-swap');
let hls;
async function start() {
@@ -147,6 +173,8 @@ async function start() {
reset_audio_el.disabled = true;
resolution_el.disabled = true;
waiting_el.classList.remove('d-none');
mic_el.removeEventListener('click', mic_save);
camera_el.removeEventListener('click', camera_save);
collapse_nav();
@@ -166,7 +194,7 @@ async function start() {
const zoom_video = zoom_video_el.checked;
const lock_portrait = /*screen.orientation.type.startsWith('portrait') &&*/ lock_portrait_el.checked;
let video_el, silence, audio_source, audio_dest, gl_canvas, canvas_stream, camera_stream, done = false;
let video_el, camera_on = false, silence, audio_source, audio_dest, gl_canvas, canvas_stream, camera_stream, done = false;
function cleanup(err) {
if (err) {
@@ -176,6 +204,20 @@ async function start() {
return;
}
done = true;
mic_el.removeEventListener('click', mic_toggle);
if (!!localStorage.getItem('streamana-mic-on')) {
mic_icon_el.classList.remove('off');
} else {
mic_icon_el.classList.add('off');
}
mic_el.addEventListener('click', mic_save);
camera_el.removeEventListener('click', camera_toggle);
if (!!localStorage.getItem('streamana-camera-on')) {
camera_icon_el.classList.remove('off');
} else {
camera_icon_el.classList.add('off');
}
camera_el.addEventListener('click', camera_save);
greyscale_el.removeEventListener('input', greyscale);
camera_swap_el.classList.add('d-none');
camera_swap_el.removeEventListener('click', about_face);
@@ -227,7 +269,9 @@ async function start() {
function update() {
// update the canvas
if ((video_el.videoWidth > 0) &&
if (!camera_on) {
gl_canvas.onLoop();
} else if ((video_el.videoWidth > 0) &&
(video_el.videoHeight > 0) &&
gl_canvas.onLoop()) {
// get aspect ratio of video
@@ -288,25 +332,29 @@ async function start() {
canvas_el.style.width = `${width}px`;
canvas_el.style.height = `${height}px`;
// TODO:
// select which camera to use (front/rear)?
// option to switch audio as well
// audio source has two channels even though media stream is mono - check what's recorded
// allow select audio and video devices
// await navigator.mediaDevices.enumerateDevices()
// mute option +
// hide camera option |<- these input list along with audio and video devices?
// audio/video source (url/file) option +
// option to mix in >1 audio?
// allow audio streaming only
// scheduling (e.g. pre-roll)?
// loop? (e.g. off-air image or video loop?)
// windows, iOS, find a mac to test
// performance on mobile
// a40 no buffers currently available in the reader queue
// windows, android, iOS, find a mac to test
}
}
async function start_camera(requested_facing_mode) {
mic_el.removeEventListener('click', mic_toggle);
camera_el.removeEventListener('click', camera_toggle);
camera_swap_el.removeEventListener('click', about_face);
async function finish() {
await hls.start();
mic_el.addEventListener('click', mic_toggle);
camera_el.addEventListener('click', camera_toggle);
camera_swap_el.addEventListener('click', about_face);
}
const need_audio = (audio_source === silence) && !mic_icon_el.classList.contains('off');
const need_video = !camera_on && !camera_icon_el.classList.contains('off');
if (!need_audio && !need_video) {
return await finish();
}
const camera_video_constraints = {
width: video_encoder_config.width,
height: video_encoder_config.height,
@@ -319,11 +367,11 @@ async function start() {
try {
camera_stream = await navigator.mediaDevices.getUserMedia({
audio: audio_source === silence,
video: camera_video_constraints
audio: need_audio,
video: need_video ? camera_video_constraints : false
});
} catch (ex) {
if (audio_source !== silence) {
if (!need_audio) {
throw ex;
}
// retry in case audio isn't available
@@ -334,20 +382,28 @@ async function start() {
});
}
const video_settings = camera_stream.getVideoTracks()[0].getSettings();
facing_mode = video_settings.facingMode || 'user';
localStorage.setItem('streamana-facing-mode', facing_mode);
// wait for video to load (must come after gl_canvas.setTexture() since it
// wait for stream to load (must come after gl_canvas.setTexture() since it
// registers a loadeddata handler which then registers a play handler)
video_el.addEventListener('loadeddata', async function () {
try {
console.log(`video resolution: ${this.videoWidth}x${this.videoHeight}`);
// start the camera video
// start the stream
this.play();
if (audio_source === silence) {
if (need_video) {
const video_tracks = camera_stream.getVideoTracks();
if (video_tracks.length > 0) {
facing_mode = video_tracks[0].facingMode || 'user';
localStorage.setItem('streamana-facing-mode', facing_mode);
camera_on = true;
gl_canvas.setUniform('u_active', camera_on);
} else {
console.warn("No video present");
}
}
if (need_audio) {
if (camera_stream.getAudioTracks().length > 0) {
audio_source.disconnect();
// add audio if present
@@ -358,9 +414,7 @@ async function start() {
}
}
await hls.start();
camera_swap_el.addEventListener('click', about_face);
await finish();
} catch (ex) {
cleanup(ex);
}
@@ -371,8 +425,7 @@ async function start() {
}
function about_face() {
camera_swap_el.removeEventListener('click', about_face);
// need to set camera_on in here
if (camera_stream) {
for (let track of camera_stream.getVideoTracks()) {
track.stop();
@@ -384,6 +437,7 @@ async function start() {
if (audio_source !== silence) {
audio_source.disconnect();
audio_source = silence;
audio_source.connect(audio_dest);
}
}
}
@@ -395,6 +449,39 @@ async function start() {
gl_canvas.setUniform('u_greyscale', this.checked);
}
function mic_toggle() {
if (mic_icon_el.classList.contains('off')) {
if (audio_source !== silence) {
if (camera_stream) {
for (let track of camera_stream.getAudioTracks()) {
track.stop();
}
}
audio_source.disconnect();
audio_source = silence;
audio_source.connect(audio_dest);
}
}
start_camera(facing_mode);
}
function camera_toggle() {
if (camera_icon_el.classList.contains('off')) {
if (camera_on) {
if (camera_stream) {
for (let track of camera_stream.getVideoTracks()) {
track.stop();
}
}
camera_on = false;
gl_canvas.setUniform('u_active', camera_on);
}
}
start_camera(facing_mode);
}
try {
// create video element which will be used for grabbing the frames to
// write to a canvas so we can apply webgl shaders
@@ -417,7 +504,6 @@ async function start() {
// use glsl-canvas to make managing webgl stuff easier
gl_canvas = new GlCanvas(canvas_el, {
// as an example, greyscale the stream
fragmentString: shader
});
@@ -430,6 +516,9 @@ async function start() {
gl_canvas.setUniform('u_greyscale', greyscale_el.checked);
greyscale_el.addEventListener('input', greyscale);
// tell shader camera hasn't started
gl_canvas.setUniform('u_active', camera_on);
// check whether we're locking portrait mode or zooming (display without bars)
if (lock_portrait) {
// rotate the canvas