From 3ad1f14c2524c9bedcba1b1feb6986b14f0da247 Mon Sep 17 00:00:00 2001 From: Goffi Date: Sat, 8 Nov 2025 16:05:54 +0100 Subject: [PATCH] frontends (tools/Webrtc): use specific configs for video sources: - Each video source (webcam, desktop, muted) has now a specific config, to adapt framerate, bitrate and keyframe max distance. - Improve logging. - Tuned pipeline to make it error resilient and improve bitrate. --- libervia/frontends/tools/webrtc.py | 156 +++++++++++++++++------------ 1 file changed, 92 insertions(+), 64 deletions(-) diff --git a/libervia/frontends/tools/webrtc.py b/libervia/frontends/tools/webrtc.py index 55899694..4a2ead3b 100644 --- a/libervia/frontends/tools/webrtc.py +++ b/libervia/frontends/tools/webrtc.py @@ -35,7 +35,7 @@ import asyncio from datetime import datetime import logging import re -from typing import Callable, Final +from typing import Callable, Final, NamedTuple from urllib.parse import quote_plus from libervia.backend.tools.common import data_format @@ -68,6 +68,20 @@ VIDEO_SOURCE_AUTO: Final = "v4l2src" AUDIO_SOURCE_AUTO: Final = "pulsesrc" NONE_NOT_IMPLEMENTED_MSG: Final = "None value is not handled yet." +class VideoConfig(NamedTuple): + fps: int + bitrate: int + keyframe_max: int + +WEBCAM = VideoConfig(fps=24, bitrate=1200000, keyframe_max=60) +DESKTOP = VideoConfig(fps=15, bitrate=800000, keyframe_max=30) +MUTED = VideoConfig(fps=24, bitrate=100000, keyframe_max=60) +VIDEO_CONFIG = { + "video": WEBCAM, + "desktop": DESKTOP, + "muted": MUTED, +} + Gst.init(None) @@ -113,7 +127,7 @@ class WebRTC: Only one video stream is then produced (the local one). If None, PiP mode is selected automatically according to selected sink (it's used for SinksAuto only for now). - @param target_size: Expected size of the final sink stream. Mainly use by composer + @param target_size: Expected size of the final sink stream. Mainly used by composer when ``merge_pip`` is set. None to autodetect (no real autodetection implemeted yet, default to (1280,720)). @@ -273,8 +287,8 @@ class WebRTC: """Generate Dot File for debugging ``GST_DEBUG_DUMP_DOT_DIR`` environment variable must be set to destination dir. - ``dot -Tpng -o .png .dot`` can be use to convert to a PNG file. - See + ``dot -Tpng -o .png .dot`` can be used to convert to a PNG + file. See https://gstreamer.freedesktop.org/documentation/gstreamer/debugutils.html?gi-language=python#GstDebugGraphDetails for details. @@ -583,7 +597,7 @@ class WebRTC: gst_pipe_elements.append( "compositor name=compositor background=black " f"! video/x-raw,width={self.target_width}," - f"height={self.target_height},framerate=30/1 " + f"height={self.target_height},framerate={WEBCAM.fps}/1 " f"! {local_video_sink_elt}" ) local_video_sink_elt = "compositor.sink_1" @@ -594,7 +608,6 @@ class WebRTC: gst_pipe_elements.append( f""" input-selector name=video_selector - ! video/x-raw,framerate=30/1 ! tee name=t {video_source_elt} name=video_src @@ -607,10 +620,11 @@ class WebRTC: t. ! queue max-size-buffers=3 leaky=downstream - ! videoscale + ! videorate + ! capsfilter name=main_rate_caps caps=video/x-raw,framerate={WEBCAM.fps}/1 ! videoconvert - ! vp8enc deadline=1 keyframe-max-dist=30 cpu-used=5 target-bitrate=1000000 - ! rtpvp8pay picture-id-mode=15-bit + ! vp8enc name=vp8enc deadline=1 keyframe-max-dist={WEBCAM.keyframe_max} cpu-used=5 target-bitrate={WEBCAM.bitrate} error-resilient=1 + ! rtpvp8pay picture-id-mode=15-bit mtu=1200 ! application/x-rtp,media=video,encoding-name=VP8,payload={video_pt} ! sendrecv. """ @@ -636,7 +650,7 @@ class WebRTC: ! queue max-size-buffers=10 max-size-time=0 max-size-bytes=0 leaky=downstream ! audioconvert ! audioresample - ! opusenc audio-type=voice + ! opusenc audio-type=voice bitrate=32000 ! rtpopuspay ! application/x-rtp,media=audio,encoding-name=OPUS,payload={audio_pt} ! sendrecv. @@ -804,7 +818,7 @@ class WebRTC: def on_notify_connection_state(pspect, param): state = self.webrtcbin.get_property("connection-state") - log.info(f"WebRTC connection state: {state}") + log.debug(f"WebRTC connection state: {state}") if state == GstWebRTC.WebRTCPeerConnectionState.FAILED: log.error("WebRTC connection failed") @@ -828,7 +842,7 @@ class WebRTC: def on_offer_created(self, promise): """Callback for when SDP offer is created.""" - log.info("on_offer_created called") + log.debug("on_offer_created called") assert promise.wait() == Gst.PromiseResult.REPLIED reply = promise.get_reply() if reply is None: @@ -836,7 +850,7 @@ class WebRTC: return offer = reply["offer"] self.offer = offer.sdp.as_text() - log.info(f"SDP offer created: \n{self.offer}") + log.debug(f"SDP offer created: \n{self.offer}") self._set_media_types(self.offer) promise = Gst.Promise.new() self.webrtcbin.emit("set-local-description", offer, promise) @@ -855,7 +869,7 @@ class WebRTC: self.webrtcbin.emit("set-local-description", answer, promise) promise.interrupt() answer_sdp = answer.sdp.as_text() - log.info(f"SDP answer set: \n{answer_sdp}") + log.debug(f"SDP answer set: \n{answer_sdp}") self.sdp_set = True self._a_call(self.bridge.call_answer_sdp, self.sid, answer_sdp, self.profile) @@ -935,14 +949,14 @@ class WebRTC: # Check and log the original size of the video width = self.target_width height = self.target_height - log.info(f"Original video size: {width}x{height}") + log.debug(f"Original video size: {width}x{height}") # This is a fix for an issue found with Movim on desktop: a non standard # resolution is used (990x557) resulting in bad alignement and no color in # rendered image adjust_resolution = width % 4 != 0 or height % 4 != 0 if adjust_resolution: - log.info("non standard resolution, we need to adjust size") + log.warning("non standard resolution, we need to adjust size") width = (width + 3) // 4 * 4 height = (height + 3) // 4 * 4 log.info(f"Adjusted video size: {width}x{height}") @@ -1117,7 +1131,7 @@ class WebRTC: link_ret = pad.link(decodebin.get_static_pad("sink")) if link_ret == Gst.PadLinkReturn.OK: - log.info(f"Successfully linked pad '{pad.get_name()}' to a new decodebin.") + log.debug(f"Successfully linked pad '{pad.get_name()}' to a new decodebin.") elif link_ret == Gst.PadLinkReturn.WAS_LINKED: log.debug(f"Pad '{pad.get_name()}' was already linked. Cleaning up redundant decodebin.") # The pad is already handled. We must clean up the decodebin we just created. @@ -1362,7 +1376,7 @@ class WebRTC: def on_ice_connection_state(self, pspec, __): state = self.webrtcbin.props.ice_connection_state - log.info(f"ICE connection state changed to: {state}") + log.debug(f"ICE connection state changed to: {state}") if state == GstWebRTC.WebRTCICEConnectionState.FAILED: log.error("ICE connection failed - check firewall/STUN/TURN configuration") @@ -1453,15 +1467,33 @@ class WebRTC: @param source: 'desktop', 'video', 'muted' or None for muted source. """ + assert self.pipeline is not None if source is None: source = "muted" - if source not in ["video", "muted", "desktop"]: + if source not in VIDEO_CONFIG: raise ValueError( - f"Invalid source: {source!r}, use one of {'video', 'muted', 'desktop'}" + f"Invalid source: {source!r}, use one of {VIDEO_CONFIG.keys()}" ) self.set_pipeline_state(Gst.State.PAUSED) + vp8enc = self.pipeline.get_by_name("vp8enc") + if vp8enc is None: + raise exceptions.InternalError("Can't find vp8enc, it should be here.") + + rate_caps_filter = self.pipeline.get_by_name("main_rate_caps") + if rate_caps_filter is None: + raise exceptions.InternalError("Can't find main_rate_caps, it should be here.") + + config = VIDEO_CONFIG[source] + vp8enc.set_property("target-bitrate", config.bitrate) + vp8enc.set_property("keyframe-max-dist", config.keyframe_max) + + # Update the framerate for the entire encoding pipeline + new_caps = Gst.Caps.from_string(f"video/x-raw,framerate={config.fps}/1") + rate_caps_filter.set_property("caps", new_caps) + log.debug(f"Updated pipeline framerate to {config.fps} FPS") + # Create a new desktop source if necessary if source == "desktop": self._setup_desktop_source(self.desktop_sharing_data) @@ -1500,6 +1532,7 @@ class WebRTC: @param properties: The properties to set on the desktop source. """ + log.debug("Setting up new desktop source...") source_elt = "ximagesrc" if self.desktop_portal is None else "pipewiresrc" desktop_src = Gst.ElementFactory.make(source_elt, "desktop_src") if properties is None: @@ -1507,77 +1540,72 @@ class WebRTC: for key, value in properties.items(): log.debug(f"setting {source_elt} property: {key!r}={value!r}") desktop_src.set_property(key, value) + + video_scale = Gst.ElementFactory.make("videoscale", "desktop_videoscale") video_convert = Gst.ElementFactory.make("videoconvert", "desktop_videoconvert") video_rate = Gst.ElementFactory.make("videorate", "desktop_videorate") - video_scale = Gst.ElementFactory.make("videoscale", "desktop_videoscale") - - # Add capsfilter to ensure proper format matching with video_selector - capsfilter = Gst.ElementFactory.make("capsfilter", "desktop_capsfilter") - caps = Gst.Caps.from_string("video/x-raw,framerate=30/1") - capsfilter.set_property("caps", caps) queue = Gst.ElementFactory.make("queue", "desktop_queue") queue.set_property("leaky", "downstream") queue.set_property("max-size-buffers", 3) + queue.set_property("flush-on-eos", True) - self.pipeline.add(desktop_src) - self.pipeline.add(video_convert) - self.pipeline.add(video_rate) - self.pipeline.add(video_scale) - self.pipeline.add(capsfilter) - self.pipeline.add(queue) + desktop_elements = [ + desktop_src, video_scale, video_convert, video_rate, queue + ] - # Link the elements: src → convert → rate → scale → capsfilter → queue - if not desktop_src.link(video_convert): - log.error("Failed to link desktop_src to videoconvert") - self._cleanup_desktop_elements([desktop_src, video_convert, video_rate, - video_scale, capsfilter, queue]) + for element in desktop_elements: + self.pipeline.add(element) + + # Link the elements in the following order: + # src → scale → convert → rate → queue + # This order ensures we scale the high-resolution buffer first, saving CPU, + # then convert it to a more standard format that videorate can handle easily. + if not desktop_src.link(video_scale): + log.error("Failed to link desktop_src to videoscale") + self._cleanup_desktop_elements(desktop_elements) + return + + if not video_scale.link(video_convert): + log.error("Failed to link videoscale to videoconvert") + self._cleanup_desktop_elements(desktop_elements) return if not video_convert.link(video_rate): log.error("Failed to link videoconvert to videorate") - self._cleanup_desktop_elements([desktop_src, video_convert, video_rate, - video_scale, capsfilter, queue]) + self._cleanup_desktop_elements(desktop_elements) return - if not video_rate.link(video_scale): - log.error("Failed to link videorate to videoscale") - self._cleanup_desktop_elements([desktop_src, video_convert, video_rate, - video_scale, capsfilter, queue]) - return - - if not video_scale.link(capsfilter): - log.error("Failed to link videoscale to capsfilter") - self._cleanup_desktop_elements([desktop_src, video_convert, video_rate, - video_scale, capsfilter, queue]) - return - - if not capsfilter.link(queue): - log.error("Failed to link capsfilter to queue") - self._cleanup_desktop_elements([desktop_src, video_convert, video_rate, - video_scale, capsfilter, queue]) + if not video_rate.link(queue): + log.error("Failed to link videorate to queue") + self._cleanup_desktop_elements(desktop_elements) return sink_pad_template = self.video_selector.get_pad_template("sink_%u") self.desktop_sink_pad = self.video_selector.request_pad( sink_pad_template, None, None ) + if not self.desktop_sink_pad: + log.error("Failed to request a new sink pad from video_selector.") + self._cleanup_desktop_elements(desktop_elements) + return + + log.debug(f"Requested and received pad {self.desktop_sink_pad.get_name()} from video_selector.") + queue_src_pad = queue.get_static_pad("src") link_ret = queue_src_pad.link(self.desktop_sink_pad) if link_ret != Gst.PadLinkReturn.OK: log.error(f"Failed to link queue to video_selector: {link_ret}") - self._cleanup_desktop_elements([desktop_src, video_convert, video_rate, - video_scale, capsfilter, queue]) + self._cleanup_desktop_elements(desktop_elements) + self.video_selector.release_request_pad(self.desktop_sink_pad) + self.desktop_sink_pad = None return - desktop_src.sync_state_with_parent() - video_convert.sync_state_with_parent() - video_rate.sync_state_with_parent() - video_scale.sync_state_with_parent() - capsfilter.sync_state_with_parent() - queue.sync_state_with_parent() + # Sync states after successful linking + for element in desktop_elements: + element.sync_state_with_parent() - log.info("Desktop source pipeline successfully created and linked") + log.debug("Desktop source pipeline successfully created and linked.") def _cleanup_desktop_elements(self, elements: list[Gst.Element]) -> None: """Helper to cleanup elements on error.