diff --git a/.gitignore b/.gitignore index e66a907..e95701f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ *~ -data/*.pem -galene -galene-password-generator/galene-password-generator -passwd -groups/*.json -static/*.d.ts +/galene +/galenectl/galenectl +/data +/groups/**/*.json +/static/**/*.d.ts +/static/third-party/tasks-vision diff --git a/CHANGES b/CHANGES index e27e691..ee96ba7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +Galene 0.96 (unreleased): + + * Implemented asynchronous filters. + * Implemented background blur. + * More fixes to RTCP feedback signalling (thanks to Sean DuBois). + 4 December 2024: Galene 0.95 * Galene now requires Go 1.20. diff --git a/go.mod b/go.mod index b7ce938..c2a758a 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,11 @@ require ( github.com/jech/cert v0.0.0-20240301122532-f491cf43a77d github.com/jech/samplebuilder v0.0.0-20221109182433-6cbba09fc1c9 github.com/pion/ice/v2 v2.3.37 - github.com/pion/rtcp v1.2.14 + github.com/pion/rtcp v1.2.15 github.com/pion/rtp v1.8.9 github.com/pion/sdp/v3 v3.0.9 github.com/pion/turn/v2 v2.1.6 - github.com/pion/webrtc/v3 v3.3.4 + github.com/pion/webrtc/v3 v3.3.5 golang.org/x/crypto v0.31.0 golang.org/x/sys v0.28.0 golang.org/x/term v0.27.0 @@ -22,18 +22,19 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/pion/datachannel v1.5.9 // indirect + github.com/pion/datachannel v1.5.10 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect github.com/pion/interceptor v0.1.37 // indirect github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.12 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/sctp v1.8.33 // indirect + github.com/pion/sctp v1.8.35 // indirect github.com/pion/srtp/v2 v2.0.20 // indirect github.com/pion/stun v0.6.1 // indirect github.com/pion/transport/v2 v2.2.10 // indirect + github.com/pion/transport/v3 v3.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/testify v1.9.0 // indirect + github.com/stretchr/testify v1.10.0 // indirect github.com/wlynxg/anet v0.0.5 // indirect golang.org/x/net v0.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 8eeae4c..41315ef 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg= github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg= -github.com/pion/datachannel v1.5.9 h1:LpIWAOYPyDrXtU+BW7X0Yt/vGtYxtXQ8ql7dFfYUVZA= -github.com/pion/datachannel v1.5.9/go.mod h1:kDUuk4CU4Uxp82NH4LQZbISULkX/HtzKa4P7ldf9izE= +github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o= +github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M= github.com/pion/dtls/v2 v2.0.9/go.mod h1:O0Wr7si/Zj5/EBFlDzDd6UtVxx25CE1r7XM7BQKYQho= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= @@ -66,8 +66,8 @@ github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TB github.com/pion/rtcp v1.2.6/go.mod h1:52rMNPWFsjr39z9B9MhnkqhPLoeHTv1aN63o/42bWE0= github.com/pion/rtcp v1.2.8/go.mod h1:qVPhiCzAm4D/rxb6XzKeyZiQK69yJpbUDJSF7TgrqNo= github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= -github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= -github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= +github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= +github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= github.com/pion/rtp v1.7.0/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= github.com/pion/rtp v1.7.2/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= @@ -75,8 +75,8 @@ github.com/pion/rtp v1.8.9 h1:E2HX740TZKaqdcPmf4pw6ZZuG8u5RlMMt+l3dxeu6Wk= github.com/pion/rtp v1.8.9/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0= github.com/pion/sctp v1.7.12/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s= -github.com/pion/sctp v1.8.33 h1:dSE4wX6uTJBcNm8+YlMg7lw1wqyKHggsP5uKbdj+NZw= -github.com/pion/sctp v1.8.33/go.mod h1:beTnqSzewI53KWoG3nqB282oDMGrhNxBdb+JZnkCwRM= +github.com/pion/sctp v1.8.35 h1:qwtKvNK1Wc5tHMIYgTDJhfZk7vATGVHhXbUDfHbYwzA= +github.com/pion/sctp v1.8.35/go.mod h1:EcXP8zCYVTRy3W9xtOF7wJm1L1aXfKRQzaM33SjQlzg= github.com/pion/sdp/v3 v3.0.4/go.mod h1:bNiSknmJE0HYBprTHXKPQ3+JjacTv5uap92ueJZKsRk= github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= @@ -88,7 +88,6 @@ github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A= github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q= -github.com/pion/transport v0.12.3 h1:vdBfvfU/0Wq8kd2yhUMSDB/x+O4Z9MYVl2fJ5BT4JZw= github.com/pion/transport v0.12.3/go.mod h1:OViWW9SP2peE/HbwBvARicmAVnesphkNkCVZIWJ6q9A= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= @@ -97,14 +96,15 @@ github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQp github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= +github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= github.com/pion/turn/v2 v2.0.5/go.mod h1:APg43CFyt/14Uy7heYUOGWdkem/Wu4PhCO/bjyrTqMw= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M= github.com/pion/webrtc/v3 v3.1.0/go.mod h1:t51XSam1k56eYLuO1Ubxjs3pDBfGYxkGBFhYf55Mn/s= -github.com/pion/webrtc/v3 v3.3.4 h1:v2heQVnXTSqNRXcaFQVOhIOYkLMxOu1iJG8uy1djvkk= -github.com/pion/webrtc/v3 v3.3.4/go.mod h1:liNa+E1iwyzyXqNUwvoMRNQ10x8h8FOeJKL8RkIbamE= +github.com/pion/webrtc/v3 v3.3.5 h1:ZsSzaMz/i9nblPdiAkZoP+E6Kmjw+jnyq3bEmU3EtRg= +github.com/pion/webrtc/v3 v3.3.5/go.mod h1:liNa+E1iwyzyXqNUwvoMRNQ10x8h8FOeJKL8RkIbamE= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -120,8 +120,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= diff --git a/static/background-blur-worker.js b/static/background-blur-worker.js index 5c7ff75..e4e840a 100644 --- a/static/background-blur-worker.js +++ b/static/background-blur-worker.js @@ -1,18 +1,36 @@ +// Copyright (c) 2024 by Juliusz Chroboczek. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + 'use strict'; let imageSegmenter; -async function loadImageSegmenter() { +async function loadImageSegmenter(model) { let module = await import('/third-party/tasks-vision/vision_bundle.mjs'); - let vision = await module.FilesetResolver.forVisionTasks( "/third-party/tasks-vision/wasm" ); - imageSegmenter = - await module.ImageSegmenter.createFromOptions(vision, { + return await module.ImageSegmenter.createFromOptions(vision, { baseOptions: { - modelAssetPath: '/third-party/tasks-vision/models/selfie_segmenter.tflite', + modelAssetPath: model, }, outputCategoryMask: true, outputConfidenceMasks: false, @@ -20,19 +38,24 @@ async function loadImageSegmenter() { }); } -loadImageSegmenter(); - -onmessage = e => { - let bitmap = e.data.bitmap; - if(!(bitmap instanceof ImageBitmap)) { - postMessage(new Error('Bad type for worker data')); +onmessage = async e => { + let data = e.data; + if(imageSegmenter == null) { + try { + imageSegmenter = await loadImageSegmenter(data.model); + if(imageSegmenter == null) + throw new Error("loadImageSegmenter returned null"); + } catch(e) { + postMessage(e); + return; + } + postMessage(null); return; } - if(!imageSegmenter) { - // not ready yet - bitmap.close(); - postMessage(null); + let bitmap = e.data.bitmap; + if(!(bitmap instanceof ImageBitmap)) { + postMessage(new Error('Bad type for worker data')); return; } diff --git a/static/galene.js b/static/galene.js index f64d094..53ad536 100644 --- a/static/galene.js +++ b/static/galene.js @@ -1188,6 +1188,30 @@ async function setFilter(c) { c.userdata.filter = filter; } +/** + * Sends a message to a worker, then waits for a reply. + * + * @param {Worker} worker + * @param {any} message + * @param {any[]} [transfer] + */ +async function workerSendReceive(worker, message, transfer) { + let p = new Promise((resolve, reject) => { + worker.onmessage = e => { + if(e && e.data) { + if(e.data instanceof Error) + reject(e.data); + else + resolve(e.data); + } else { + resolve(null); + } + }; + }); + worker.postMessage(message, transfer); + return await p +} + /** * @type {Object.} */ @@ -1227,14 +1251,24 @@ let filters = { 'background-blur': { description: 'Background blur', predicate: async function() { + if(isSafari()) { + console.warn( + 'Background blur does not work on Safari, disabled.' + ); + return false; + + } let r = await fetch('/third-party/tasks-vision/vision_bundle.mjs', { method: 'HEAD', }); - if(!r.ok && r.status !== 404) - console.warn( - `Fetch vision_bundle.mjs: ${r.status} ${r.statusText}`, - ); - return r.ok; + if(!r.ok) { + if(r.status !== 404) + console.warn( + `Fetch vision_bundle.mjs: ${r.status} ${r.statusText}`, + ); + return false; + } + return true; }, init: async function(ctx) { if(!(this instanceof Filter)) @@ -1242,6 +1276,9 @@ let filters = { if(this.userdata.worker) throw new Error("Worker already running (this shouldn't happen)") this.userdata.worker = new Worker('/background-blur-worker.js'); + await workerSendReceive(this.userdata.worker, { + model: '/third-party/tasks-vision/models/selfie_segmenter.tflite', + }); }, cleanup: async function() { if(this.userdata.worker.onmessage) { @@ -1253,23 +1290,10 @@ let filters = { draw: async function(src, ctx) { let bitmap = await createImageBitmap(src); try { - let p = new Promise((resolve, reject) => { - this.userdata.worker.onmessage = e => { - if(e && e.data) { - if(e.data instanceof Error) - reject(e.data); - else - resolve(e.data); - } else { - resolve(null); - } - }; - }); - this.userdata.worker.postMessage({ + let result = await workerSendReceive(this.userdata.worker, { bitmap: bitmap, timestamp: performance.now(), }, [bitmap]); - let result = await p; if(!result) return false; @@ -1369,12 +1393,7 @@ async function setUpStream(c, stream) { c.setStream(stream); - try { - await setFilter(c); - } catch(e) { - displayWarning("Couldn't set filter: " + e); - } - + // set up the handler early, in case setFilter fails. c.onclose = async replace => { await removeFilter(c); if(!replace) { @@ -1385,6 +1404,8 @@ async function setUpStream(c, stream) { } } + await setFilter(c); + /** * @param {MediaStreamTrack} t */ @@ -1507,10 +1528,20 @@ async function replaceUpStream(c) { cn.userdata.onclose = c.userdata.onclose; let media = /** @type{HTMLVideoElement} */ (document.getElementById('media-' + c.localId)); - await setUpStream(cn, c.stream); + try { + await setUpStream(cn, c.stream); + } catch(e) { + console.error(e); + displayError(e); + cn.close(); + c.close(); + return null; + } + await setMedia(cn, cn.label == 'camera' && getSettings().mirrorView, cn.label == 'video' && media); + return cn; } @@ -1611,8 +1642,14 @@ async function addLocalMedia(localId) { displayWarning(`Unknown filter ${settings.filter}`); } - await setUpStream(c, stream); - await setMedia(c, settings.mirrorView); + try { + await setUpStream(c, stream); + await setMedia(c, settings.mirrorView); + } catch(e) { + console.error(e); + displayError(e); + c.close(); + } setButtonsVisibility(); } @@ -1734,6 +1771,8 @@ function closeUpMedia(label) { * @returns {Stream} */ function findUpMedia(label) { + if(!serverConnection) + return null; for(let id in serverConnection.up) { let c = serverConnection.up[id]; if(c.label === label) @@ -4263,7 +4302,7 @@ async function start() { window.history.replaceState(null, '', window.location.pathname); setTitle(groupStatus.displayName || capitalise(group)); - await addFilters(); + addFilters(); await setMediaChoices(false); reflectSettings();