frontends (tools/webrtc): Add a video recovery mecanism:
When packets are lost, artefacts appear and the remove stream video quickly becomes mush (can be reproduced by blocking UDP for few seconds). To recover from this situation, the video pad is monitored for GAP events, and when they occur, a new keyframe is requested 1s after the last GAP.
This commit is contained in:
parent
3ad1f14c25
commit
ba7ea7664a
1 changed files with 78 additions and 1 deletions
|
|
@ -20,7 +20,7 @@ from collections.abc import Awaitable
|
|||
import gi
|
||||
|
||||
gi.require_versions({"Gst": "1.0", "GstWebRTC": "1.0"})
|
||||
from gi.repository import Gst, GstWebRTC, GstSdp
|
||||
from gi.repository import Gst, GstWebRTC, GstSdp, GLib
|
||||
|
||||
from libervia.backend.core import exceptions
|
||||
|
||||
|
|
@ -180,6 +180,7 @@ class WebRTC:
|
|||
)
|
||||
else:
|
||||
self.desktop_portal = None
|
||||
self._gap_recovery_timeout = None
|
||||
self.reset_instance()
|
||||
|
||||
@property
|
||||
|
|
@ -501,6 +502,9 @@ class WebRTC:
|
|||
if self.reset_cb is not None:
|
||||
self.reset_cb()
|
||||
self.data_channels: dict[str, GstWebRTC.WebRTCDataChannel] = {}
|
||||
if self._gap_recovery_timeout is not None:
|
||||
GLib.source_remove(self._gap_recovery_timeout)
|
||||
self._gap_recovery_timeout = None
|
||||
|
||||
@property
|
||||
def data_channel(self) -> GstWebRTC.WebRTCDataChannel:
|
||||
|
|
@ -921,6 +925,73 @@ class WebRTC:
|
|||
new_height = int(max_width / aspect_ratio)
|
||||
return max_width, new_height
|
||||
|
||||
def on_video_pad_probe(
|
||||
self,
|
||||
pad: Gst.Pad,
|
||||
info: Gst.PadProbeInfo
|
||||
) -> Gst.PadProbeReturn:
|
||||
"""Monitor video pad for GAP events indicating packet loss.
|
||||
|
||||
When gaps are detected, mark that we need recovery. After gaps stop,
|
||||
wait a bit then request a keyframe.
|
||||
"""
|
||||
event = info.get_event()
|
||||
if event and event.type == Gst.EventType.GAP:
|
||||
log.debug("Video packet loss detected (GAP event).")
|
||||
|
||||
if self._gap_recovery_timeout is not None:
|
||||
GLib.source_remove(self._gap_recovery_timeout)
|
||||
self._gap_recovery_timeout = None
|
||||
|
||||
# Schedule keyframe request after gap stops.
|
||||
# Wait 1 second after last GAP event to ensure connection has recovered
|
||||
self._gap_recovery_timeout = GLib.timeout_add_seconds(
|
||||
1, self._on_gap_recovery
|
||||
)
|
||||
|
||||
return Gst.PadProbeReturn.OK
|
||||
|
||||
def _on_gap_recovery(self) -> bool:
|
||||
"""Called after GAP events have stopped, indicating connection recovery.
|
||||
|
||||
@return: False to remove the timeout (one-shot)
|
||||
"""
|
||||
log.info("Connection recovering after packet loss, requesting keyframe.")
|
||||
self._request_keyframe()
|
||||
self._gap_recovery_timeout = None
|
||||
# We return False to make it a one-short timer callback.
|
||||
return False
|
||||
|
||||
def _request_keyframe(self) -> None:
|
||||
"""Requests a keyframe from the remote peer.
|
||||
|
||||
A keyframe is requested by sending a GstForceKeyUnit event upstream. This is used
|
||||
for video recovery after packet loss.
|
||||
"""
|
||||
if not self._remote_video_pad:
|
||||
log.warning("Cannot request keyframe, remote video pad not yet available.")
|
||||
return
|
||||
|
||||
decodebin = self._remote_video_pad.get_parent()
|
||||
if not decodebin or not isinstance(decodebin, Gst.Element):
|
||||
log.warning("Could not get parent element (decodebin) from remote video pad.")
|
||||
return
|
||||
|
||||
log.debug("Requesting keyframe from remote peer (sending PLI/FIR).")
|
||||
|
||||
# Create the GstForceKeyUnit event. This event, when it reaches the
|
||||
# rtpjitterbuffer inside webrtcbin, will cause it to send an
|
||||
# RTCP Picture Loss Indication (PLI) or Full Intra Request (FIR)
|
||||
# packet to the sender.
|
||||
structure = Gst.Structure.new_from_string("GstForceKeyUnit")
|
||||
structure.set_value("all-headers", True)
|
||||
event = Gst.Event.new_custom(Gst.EventType.CUSTOM_UPSTREAM, structure)
|
||||
|
||||
if decodebin.send_event(event):
|
||||
log.debug("Successfully sent GstForceKeyUnit event to decodebin.")
|
||||
else:
|
||||
log.error("Failed to send GstForceKeyUnit event to decodebin.")
|
||||
|
||||
def on_remote_decodebin_stream(self, decodebin, pad: Gst.Pad) -> None:
|
||||
"""Handle the stream from the remote decodebin.
|
||||
|
||||
|
|
@ -946,6 +1017,12 @@ class WebRTC:
|
|||
|
||||
self._remote_video_pad = pad
|
||||
|
||||
# Monitor for events to detect packet loss/corruption
|
||||
pad.add_probe(
|
||||
Gst.PadProbeType.EVENT_DOWNSTREAM,
|
||||
self.on_video_pad_probe
|
||||
)
|
||||
|
||||
# Check and log the original size of the video
|
||||
width = self.target_width
|
||||
height = self.target_height
|
||||
|
|
|
|||
Loading…
Reference in a new issue