frontends (tools/Webrtc): use specific configs for video sources:
- Each video source (webcam, desktop, muted) has now a specific config, to adapt framerate, bitrate and keyframe max distance. - Improve logging. - Tuned pipeline to make it error resilient and improve bitrate.
This commit is contained in:
Goffi
parent
450ea7b381
commit
3ad1f14c25
1 changed files with 92 additions and 64 deletions
|
|
@ -35,7 +35,7 @@ import asyncio
|
|||
from datetime import datetime
|
||||
import logging
|
||||
import re
|
||||
from typing import Callable, Final
|
||||
from typing import Callable, Final, NamedTuple
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from libervia.backend.tools.common import data_format
|
||||
|
|
@ -68,6 +68,20 @@ VIDEO_SOURCE_AUTO: Final = "v4l2src"
|
|||
AUDIO_SOURCE_AUTO: Final = "pulsesrc"
|
||||
NONE_NOT_IMPLEMENTED_MSG: Final = "None value is not handled yet."
|
||||
|
||||
class VideoConfig(NamedTuple):
|
||||
fps: int
|
||||
bitrate: int
|
||||
keyframe_max: int
|
||||
|
||||
WEBCAM = VideoConfig(fps=24, bitrate=1200000, keyframe_max=60)
|
||||
DESKTOP = VideoConfig(fps=15, bitrate=800000, keyframe_max=30)
|
||||
MUTED = VideoConfig(fps=24, bitrate=100000, keyframe_max=60)
|
||||
VIDEO_CONFIG = {
|
||||
"video": WEBCAM,
|
||||
"desktop": DESKTOP,
|
||||
"muted": MUTED,
|
||||
}
|
||||
|
||||
Gst.init(None)
|
||||
|
||||
|
||||
|
|
@ -113,7 +127,7 @@ class WebRTC:
|
|||
Only one video stream is then produced (the local one).
|
||||
If None, PiP mode is selected automatically according to selected sink (it's
|
||||
used for SinksAuto only for now).
|
||||
@param target_size: Expected size of the final sink stream. Mainly use by composer
|
||||
@param target_size: Expected size of the final sink stream. Mainly used by composer
|
||||
when ``merge_pip`` is set.
|
||||
None to autodetect (no real autodetection implemeted yet, default to
|
||||
(1280,720)).
|
||||
|
|
@ -273,8 +287,8 @@ class WebRTC:
|
|||
"""Generate Dot File for debugging
|
||||
|
||||
``GST_DEBUG_DUMP_DOT_DIR`` environment variable must be set to destination dir.
|
||||
``dot -Tpng -o <filename>.png <filename>.dot`` can be use to convert to a PNG file.
|
||||
See
|
||||
``dot -Tpng -o <filename>.png <filename>.dot`` can be used to convert to a PNG
|
||||
file. See
|
||||
https://gstreamer.freedesktop.org/documentation/gstreamer/debugutils.html?gi-language=python#GstDebugGraphDetails
|
||||
for details.
|
||||
|
||||
|
|
@ -583,7 +597,7 @@ class WebRTC:
|
|||
gst_pipe_elements.append(
|
||||
"compositor name=compositor background=black "
|
||||
f"! video/x-raw,width={self.target_width},"
|
||||
f"height={self.target_height},framerate=30/1 "
|
||||
f"height={self.target_height},framerate={WEBCAM.fps}/1 "
|
||||
f"! {local_video_sink_elt}"
|
||||
)
|
||||
local_video_sink_elt = "compositor.sink_1"
|
||||
|
|
@ -594,7 +608,6 @@ class WebRTC:
|
|||
gst_pipe_elements.append(
|
||||
f"""
|
||||
input-selector name=video_selector
|
||||
! video/x-raw,framerate=30/1
|
||||
! tee name=t
|
||||
|
||||
{video_source_elt} name=video_src
|
||||
|
|
@ -607,10 +620,11 @@ class WebRTC:
|
|||
|
||||
t.
|
||||
! queue max-size-buffers=3 leaky=downstream
|
||||
! videoscale
|
||||
! videorate
|
||||
! capsfilter name=main_rate_caps caps=video/x-raw,framerate={WEBCAM.fps}/1
|
||||
! videoconvert
|
||||
! vp8enc deadline=1 keyframe-max-dist=30 cpu-used=5 target-bitrate=1000000
|
||||
! rtpvp8pay picture-id-mode=15-bit
|
||||
! vp8enc name=vp8enc deadline=1 keyframe-max-dist={WEBCAM.keyframe_max} cpu-used=5 target-bitrate={WEBCAM.bitrate} error-resilient=1
|
||||
! rtpvp8pay picture-id-mode=15-bit mtu=1200
|
||||
! application/x-rtp,media=video,encoding-name=VP8,payload={video_pt}
|
||||
! sendrecv.
|
||||
"""
|
||||
|
|
@ -636,7 +650,7 @@ class WebRTC:
|
|||
! queue max-size-buffers=10 max-size-time=0 max-size-bytes=0 leaky=downstream
|
||||
! audioconvert
|
||||
! audioresample
|
||||
! opusenc audio-type=voice
|
||||
! opusenc audio-type=voice bitrate=32000
|
||||
! rtpopuspay
|
||||
! application/x-rtp,media=audio,encoding-name=OPUS,payload={audio_pt}
|
||||
! sendrecv.
|
||||
|
|
@ -804,7 +818,7 @@ class WebRTC:
|
|||
|
||||
def on_notify_connection_state(pspect, param):
|
||||
state = self.webrtcbin.get_property("connection-state")
|
||||
log.info(f"WebRTC connection state: {state}")
|
||||
log.debug(f"WebRTC connection state: {state}")
|
||||
|
||||
if state == GstWebRTC.WebRTCPeerConnectionState.FAILED:
|
||||
log.error("WebRTC connection failed")
|
||||
|
|
@ -828,7 +842,7 @@ class WebRTC:
|
|||
|
||||
def on_offer_created(self, promise):
|
||||
"""Callback for when SDP offer is created."""
|
||||
log.info("on_offer_created called")
|
||||
log.debug("on_offer_created called")
|
||||
assert promise.wait() == Gst.PromiseResult.REPLIED
|
||||
reply = promise.get_reply()
|
||||
if reply is None:
|
||||
|
|
@ -836,7 +850,7 @@ class WebRTC:
|
|||
return
|
||||
offer = reply["offer"]
|
||||
self.offer = offer.sdp.as_text()
|
||||
log.info(f"SDP offer created: \n{self.offer}")
|
||||
log.debug(f"SDP offer created: \n{self.offer}")
|
||||
self._set_media_types(self.offer)
|
||||
promise = Gst.Promise.new()
|
||||
self.webrtcbin.emit("set-local-description", offer, promise)
|
||||
|
|
@ -855,7 +869,7 @@ class WebRTC:
|
|||
self.webrtcbin.emit("set-local-description", answer, promise)
|
||||
promise.interrupt()
|
||||
answer_sdp = answer.sdp.as_text()
|
||||
log.info(f"SDP answer set: \n{answer_sdp}")
|
||||
log.debug(f"SDP answer set: \n{answer_sdp}")
|
||||
self.sdp_set = True
|
||||
self._a_call(self.bridge.call_answer_sdp, self.sid, answer_sdp, self.profile)
|
||||
|
||||
|
|
@ -935,14 +949,14 @@ class WebRTC:
|
|||
# Check and log the original size of the video
|
||||
width = self.target_width
|
||||
height = self.target_height
|
||||
log.info(f"Original video size: {width}x{height}")
|
||||
log.debug(f"Original video size: {width}x{height}")
|
||||
|
||||
# This is a fix for an issue found with Movim on desktop: a non standard
|
||||
# resolution is used (990x557) resulting in bad alignement and no color in
|
||||
# rendered image
|
||||
adjust_resolution = width % 4 != 0 or height % 4 != 0
|
||||
if adjust_resolution:
|
||||
log.info("non standard resolution, we need to adjust size")
|
||||
log.warning("non standard resolution, we need to adjust size")
|
||||
width = (width + 3) // 4 * 4
|
||||
height = (height + 3) // 4 * 4
|
||||
log.info(f"Adjusted video size: {width}x{height}")
|
||||
|
|
@ -1117,7 +1131,7 @@ class WebRTC:
|
|||
link_ret = pad.link(decodebin.get_static_pad("sink"))
|
||||
|
||||
if link_ret == Gst.PadLinkReturn.OK:
|
||||
log.info(f"Successfully linked pad '{pad.get_name()}' to a new decodebin.")
|
||||
log.debug(f"Successfully linked pad '{pad.get_name()}' to a new decodebin.")
|
||||
elif link_ret == Gst.PadLinkReturn.WAS_LINKED:
|
||||
log.debug(f"Pad '{pad.get_name()}' was already linked. Cleaning up redundant decodebin.")
|
||||
# The pad is already handled. We must clean up the decodebin we just created.
|
||||
|
|
@ -1362,7 +1376,7 @@ class WebRTC:
|
|||
|
||||
def on_ice_connection_state(self, pspec, __):
|
||||
state = self.webrtcbin.props.ice_connection_state
|
||||
log.info(f"ICE connection state changed to: {state}")
|
||||
log.debug(f"ICE connection state changed to: {state}")
|
||||
|
||||
if state == GstWebRTC.WebRTCICEConnectionState.FAILED:
|
||||
log.error("ICE connection failed - check firewall/STUN/TURN configuration")
|
||||
|
|
@ -1453,15 +1467,33 @@ class WebRTC:
|
|||
|
||||
@param source: 'desktop', 'video', 'muted' or None for muted source.
|
||||
"""
|
||||
assert self.pipeline is not None
|
||||
if source is None:
|
||||
source = "muted"
|
||||
if source not in ["video", "muted", "desktop"]:
|
||||
if source not in VIDEO_CONFIG:
|
||||
raise ValueError(
|
||||
f"Invalid source: {source!r}, use one of {'video', 'muted', 'desktop'}"
|
||||
f"Invalid source: {source!r}, use one of {VIDEO_CONFIG.keys()}"
|
||||
)
|
||||
|
||||
self.set_pipeline_state(Gst.State.PAUSED)
|
||||
|
||||
vp8enc = self.pipeline.get_by_name("vp8enc")
|
||||
if vp8enc is None:
|
||||
raise exceptions.InternalError("Can't find vp8enc, it should be here.")
|
||||
|
||||
rate_caps_filter = self.pipeline.get_by_name("main_rate_caps")
|
||||
if rate_caps_filter is None:
|
||||
raise exceptions.InternalError("Can't find main_rate_caps, it should be here.")
|
||||
|
||||
config = VIDEO_CONFIG[source]
|
||||
vp8enc.set_property("target-bitrate", config.bitrate)
|
||||
vp8enc.set_property("keyframe-max-dist", config.keyframe_max)
|
||||
|
||||
# Update the framerate for the entire encoding pipeline
|
||||
new_caps = Gst.Caps.from_string(f"video/x-raw,framerate={config.fps}/1")
|
||||
rate_caps_filter.set_property("caps", new_caps)
|
||||
log.debug(f"Updated pipeline framerate to {config.fps} FPS")
|
||||
|
||||
# Create a new desktop source if necessary
|
||||
if source == "desktop":
|
||||
self._setup_desktop_source(self.desktop_sharing_data)
|
||||
|
|
@ -1500,6 +1532,7 @@ class WebRTC:
|
|||
|
||||
@param properties: The properties to set on the desktop source.
|
||||
"""
|
||||
log.debug("Setting up new desktop source...")
|
||||
source_elt = "ximagesrc" if self.desktop_portal is None else "pipewiresrc"
|
||||
desktop_src = Gst.ElementFactory.make(source_elt, "desktop_src")
|
||||
if properties is None:
|
||||
|
|
@ -1507,77 +1540,72 @@ class WebRTC:
|
|||
for key, value in properties.items():
|
||||
log.debug(f"setting {source_elt} property: {key!r}={value!r}")
|
||||
desktop_src.set_property(key, value)
|
||||
|
||||
video_scale = Gst.ElementFactory.make("videoscale", "desktop_videoscale")
|
||||
video_convert = Gst.ElementFactory.make("videoconvert", "desktop_videoconvert")
|
||||
video_rate = Gst.ElementFactory.make("videorate", "desktop_videorate")
|
||||
video_scale = Gst.ElementFactory.make("videoscale", "desktop_videoscale")
|
||||
|
||||
# Add capsfilter to ensure proper format matching with video_selector
|
||||
capsfilter = Gst.ElementFactory.make("capsfilter", "desktop_capsfilter")
|
||||
caps = Gst.Caps.from_string("video/x-raw,framerate=30/1")
|
||||
capsfilter.set_property("caps", caps)
|
||||
|
||||
queue = Gst.ElementFactory.make("queue", "desktop_queue")
|
||||
queue.set_property("leaky", "downstream")
|
||||
queue.set_property("max-size-buffers", 3)
|
||||
queue.set_property("flush-on-eos", True)
|
||||
|
||||
self.pipeline.add(desktop_src)
|
||||
self.pipeline.add(video_convert)
|
||||
self.pipeline.add(video_rate)
|
||||
self.pipeline.add(video_scale)
|
||||
self.pipeline.add(capsfilter)
|
||||
self.pipeline.add(queue)
|
||||
desktop_elements = [
|
||||
desktop_src, video_scale, video_convert, video_rate, queue
|
||||
]
|
||||
|
||||
# Link the elements: src → convert → rate → scale → capsfilter → queue
|
||||
if not desktop_src.link(video_convert):
|
||||
log.error("Failed to link desktop_src to videoconvert")
|
||||
self._cleanup_desktop_elements([desktop_src, video_convert, video_rate,
|
||||
video_scale, capsfilter, queue])
|
||||
for element in desktop_elements:
|
||||
self.pipeline.add(element)
|
||||
|
||||
# Link the elements in the following order:
|
||||
# src → scale → convert → rate → queue
|
||||
# This order ensures we scale the high-resolution buffer first, saving CPU,
|
||||
# then convert it to a more standard format that videorate can handle easily.
|
||||
if not desktop_src.link(video_scale):
|
||||
log.error("Failed to link desktop_src to videoscale")
|
||||
self._cleanup_desktop_elements(desktop_elements)
|
||||
return
|
||||
|
||||
if not video_scale.link(video_convert):
|
||||
log.error("Failed to link videoscale to videoconvert")
|
||||
self._cleanup_desktop_elements(desktop_elements)
|
||||
return
|
||||
|
||||
if not video_convert.link(video_rate):
|
||||
log.error("Failed to link videoconvert to videorate")
|
||||
self._cleanup_desktop_elements([desktop_src, video_convert, video_rate,
|
||||
video_scale, capsfilter, queue])
|
||||
self._cleanup_desktop_elements(desktop_elements)
|
||||
return
|
||||
|
||||
if not video_rate.link(video_scale):
|
||||
log.error("Failed to link videorate to videoscale")
|
||||
self._cleanup_desktop_elements([desktop_src, video_convert, video_rate,
|
||||
video_scale, capsfilter, queue])
|
||||
return
|
||||
|
||||
if not video_scale.link(capsfilter):
|
||||
log.error("Failed to link videoscale to capsfilter")
|
||||
self._cleanup_desktop_elements([desktop_src, video_convert, video_rate,
|
||||
video_scale, capsfilter, queue])
|
||||
return
|
||||
|
||||
if not capsfilter.link(queue):
|
||||
log.error("Failed to link capsfilter to queue")
|
||||
self._cleanup_desktop_elements([desktop_src, video_convert, video_rate,
|
||||
video_scale, capsfilter, queue])
|
||||
if not video_rate.link(queue):
|
||||
log.error("Failed to link videorate to queue")
|
||||
self._cleanup_desktop_elements(desktop_elements)
|
||||
return
|
||||
|
||||
sink_pad_template = self.video_selector.get_pad_template("sink_%u")
|
||||
self.desktop_sink_pad = self.video_selector.request_pad(
|
||||
sink_pad_template, None, None
|
||||
)
|
||||
if not self.desktop_sink_pad:
|
||||
log.error("Failed to request a new sink pad from video_selector.")
|
||||
self._cleanup_desktop_elements(desktop_elements)
|
||||
return
|
||||
|
||||
log.debug(f"Requested and received pad {self.desktop_sink_pad.get_name()} from video_selector.")
|
||||
|
||||
queue_src_pad = queue.get_static_pad("src")
|
||||
link_ret = queue_src_pad.link(self.desktop_sink_pad)
|
||||
if link_ret != Gst.PadLinkReturn.OK:
|
||||
log.error(f"Failed to link queue to video_selector: {link_ret}")
|
||||
self._cleanup_desktop_elements([desktop_src, video_convert, video_rate,
|
||||
video_scale, capsfilter, queue])
|
||||
self._cleanup_desktop_elements(desktop_elements)
|
||||
self.video_selector.release_request_pad(self.desktop_sink_pad)
|
||||
self.desktop_sink_pad = None
|
||||
return
|
||||
|
||||
desktop_src.sync_state_with_parent()
|
||||
video_convert.sync_state_with_parent()
|
||||
video_rate.sync_state_with_parent()
|
||||
video_scale.sync_state_with_parent()
|
||||
capsfilter.sync_state_with_parent()
|
||||
queue.sync_state_with_parent()
|
||||
# Sync states after successful linking
|
||||
for element in desktop_elements:
|
||||
element.sync_state_with_parent()
|
||||
|
||||
log.info("Desktop source pipeline successfully created and linked")
|
||||
log.debug("Desktop source pipeline successfully created and linked.")
|
||||
|
||||
def _cleanup_desktop_elements(self, elements: list[Gst.Element]) -> None:
|
||||
"""Helper to cleanup elements on error.
|
||||
|
|
|
|||
Loading…
Reference in a new issue