frontends (tools/webrtc): Add a video recovery mecanism:

When packets are lost, artefacts appear and the remove stream video quickly becomes mush
(can be reproduced by blocking UDP for few seconds). To recover from this situation, the
video pad is monitored for GAP events, and when they occur, a new keyframe is requested 1s
after the last GAP.
This commit is contained in:
Goffi 2025-11-08 16:06:06 +01:00
commit ba7ea7664a

View file

@ -20,7 +20,7 @@ from collections.abc import Awaitable
import gi
gi.require_versions({"Gst": "1.0", "GstWebRTC": "1.0"})
from gi.repository import Gst, GstWebRTC, GstSdp
from gi.repository import Gst, GstWebRTC, GstSdp, GLib
from libervia.backend.core import exceptions
@ -180,6 +180,7 @@ class WebRTC:
)
else:
self.desktop_portal = None
self._gap_recovery_timeout = None
self.reset_instance()
@property
@ -501,6 +502,9 @@ class WebRTC:
if self.reset_cb is not None:
self.reset_cb()
self.data_channels: dict[str, GstWebRTC.WebRTCDataChannel] = {}
if self._gap_recovery_timeout is not None:
GLib.source_remove(self._gap_recovery_timeout)
self._gap_recovery_timeout = None
@property
def data_channel(self) -> GstWebRTC.WebRTCDataChannel:
@ -921,6 +925,73 @@ class WebRTC:
new_height = int(max_width / aspect_ratio)
return max_width, new_height
def on_video_pad_probe(
self,
pad: Gst.Pad,
info: Gst.PadProbeInfo
) -> Gst.PadProbeReturn:
"""Monitor video pad for GAP events indicating packet loss.
When gaps are detected, mark that we need recovery. After gaps stop,
wait a bit then request a keyframe.
"""
event = info.get_event()
if event and event.type == Gst.EventType.GAP:
log.debug("Video packet loss detected (GAP event).")
if self._gap_recovery_timeout is not None:
GLib.source_remove(self._gap_recovery_timeout)
self._gap_recovery_timeout = None
# Schedule keyframe request after gap stops.
# Wait 1 second after last GAP event to ensure connection has recovered
self._gap_recovery_timeout = GLib.timeout_add_seconds(
1, self._on_gap_recovery
)
return Gst.PadProbeReturn.OK
def _on_gap_recovery(self) -> bool:
"""Called after GAP events have stopped, indicating connection recovery.
@return: False to remove the timeout (one-shot)
"""
log.info("Connection recovering after packet loss, requesting keyframe.")
self._request_keyframe()
self._gap_recovery_timeout = None
# We return False to make it a one-short timer callback.
return False
def _request_keyframe(self) -> None:
"""Requests a keyframe from the remote peer.
A keyframe is requested by sending a GstForceKeyUnit event upstream. This is used
for video recovery after packet loss.
"""
if not self._remote_video_pad:
log.warning("Cannot request keyframe, remote video pad not yet available.")
return
decodebin = self._remote_video_pad.get_parent()
if not decodebin or not isinstance(decodebin, Gst.Element):
log.warning("Could not get parent element (decodebin) from remote video pad.")
return
log.debug("Requesting keyframe from remote peer (sending PLI/FIR).")
# Create the GstForceKeyUnit event. This event, when it reaches the
# rtpjitterbuffer inside webrtcbin, will cause it to send an
# RTCP Picture Loss Indication (PLI) or Full Intra Request (FIR)
# packet to the sender.
structure = Gst.Structure.new_from_string("GstForceKeyUnit")
structure.set_value("all-headers", True)
event = Gst.Event.new_custom(Gst.EventType.CUSTOM_UPSTREAM, structure)
if decodebin.send_event(event):
log.debug("Successfully sent GstForceKeyUnit event to decodebin.")
else:
log.error("Failed to send GstForceKeyUnit event to decodebin.")
def on_remote_decodebin_stream(self, decodebin, pad: Gst.Pad) -> None:
"""Handle the stream from the remote decodebin.
@ -946,6 +1017,12 @@ class WebRTC:
self._remote_video_pad = pad
# Monitor for events to detect packet loss/corruption
pad.add_probe(
Gst.PadProbeType.EVENT_DOWNSTREAM,
self.on_video_pad_probe
)
# Check and log the original size of the video
width = self.target_width
height = self.target_height