From ba7ea7664a951acd32a61de147cca1a4712df7e8 Mon Sep 17 00:00:00 2001
From: Goffi <goffi@goffi.org>
Date: Sat, 8 Nov 2025 16:06:06 +0100
Subject: [PATCH] frontends (tools/webrtc): Add a video recovery mecanism:

When packets are lost, artefacts appear and the remove stream video quickly becomes mush
(can be reproduced by blocking UDP for few seconds). To recover from this situation, the
video pad is monitored for GAP events, and when they occur, a new keyframe is requested 1s
after the last GAP.
---
 libervia/frontends/tools/webrtc.py | 79 +++++++++++++++++++++++++++++-
 1 file changed, 78 insertions(+), 1 deletion(-)

diff --git a/libervia/frontends/tools/webrtc.py b/libervia/frontends/tools/webrtc.py
index 4a2ead3b..63ca5a00 100644
--- a/libervia/frontends/tools/webrtc.py
+++ b/libervia/frontends/tools/webrtc.py
@@ -20,7 +20,7 @@ from collections.abc import Awaitable
 import gi
 
 gi.require_versions({"Gst": "1.0", "GstWebRTC": "1.0"})
-from gi.repository import Gst, GstWebRTC, GstSdp
+from gi.repository import Gst, GstWebRTC, GstSdp, GLib
 
 from libervia.backend.core import exceptions
 
@@ -180,6 +180,7 @@ class WebRTC:
             )
         else:
             self.desktop_portal = None
+        self._gap_recovery_timeout = None
         self.reset_instance()
 
     @property
@@ -501,6 +502,9 @@ class WebRTC:
         if self.reset_cb is not None:
             self.reset_cb()
         self.data_channels: dict[str, GstWebRTC.WebRTCDataChannel] = {}
+        if self._gap_recovery_timeout is not None:
+            GLib.source_remove(self._gap_recovery_timeout)
+            self._gap_recovery_timeout = None
 
     @property
     def data_channel(self) -> GstWebRTC.WebRTCDataChannel:
@@ -921,6 +925,73 @@ class WebRTC:
         new_height = int(max_width / aspect_ratio)
         return max_width, new_height
 
+    def on_video_pad_probe(
+        self,
+        pad: Gst.Pad,
+        info: Gst.PadProbeInfo
+    ) -> Gst.PadProbeReturn:
+        """Monitor video pad for GAP events indicating packet loss.
+
+        When gaps are detected, mark that we need recovery. After gaps stop,
+        wait a bit then request a keyframe.
+        """
+        event = info.get_event()
+        if event and event.type == Gst.EventType.GAP:
+            log.debug("Video packet loss detected (GAP event).")
+
+            if self._gap_recovery_timeout is not None:
+                GLib.source_remove(self._gap_recovery_timeout)
+                self._gap_recovery_timeout = None
+
+            # Schedule keyframe request after gap stops.
+            # Wait 1 second after last GAP event to ensure connection has recovered
+            self._gap_recovery_timeout = GLib.timeout_add_seconds(
+                1, self._on_gap_recovery
+            )
+
+        return Gst.PadProbeReturn.OK
+
+    def _on_gap_recovery(self) -> bool:
+        """Called after GAP events have stopped, indicating connection recovery.
+
+        @return: False to remove the timeout (one-shot)
+        """
+        log.info("Connection recovering after packet loss, requesting keyframe.")
+        self._request_keyframe()
+        self._gap_recovery_timeout = None
+        # We return False to make it a one-short timer callback.
+        return False
+
+    def _request_keyframe(self) -> None:
+        """Requests a keyframe from the remote peer.
+
+        A keyframe is requested by sending a GstForceKeyUnit event upstream. This is used
+        for video recovery after packet loss.
+        """
+        if not self._remote_video_pad:
+            log.warning("Cannot request keyframe, remote video pad not yet available.")
+            return
+
+        decodebin = self._remote_video_pad.get_parent()
+        if not decodebin or not isinstance(decodebin, Gst.Element):
+            log.warning("Could not get parent element (decodebin) from remote video pad.")
+            return
+
+        log.debug("Requesting keyframe from remote peer (sending PLI/FIR).")
+
+        # Create the GstForceKeyUnit event. This event, when it reaches the
+        # rtpjitterbuffer inside webrtcbin, will cause it to send an
+        # RTCP Picture Loss Indication (PLI) or Full Intra Request (FIR)
+        # packet to the sender.
+        structure = Gst.Structure.new_from_string("GstForceKeyUnit")
+        structure.set_value("all-headers", True)
+        event = Gst.Event.new_custom(Gst.EventType.CUSTOM_UPSTREAM, structure)
+
+        if decodebin.send_event(event):
+            log.debug("Successfully sent GstForceKeyUnit event to decodebin.")
+        else:
+            log.error("Failed to send GstForceKeyUnit event to decodebin.")
+
     def on_remote_decodebin_stream(self, decodebin, pad: Gst.Pad) -> None:
         """Handle the stream from the remote decodebin.
 
@@ -946,6 +1017,12 @@ class WebRTC:
 
             self._remote_video_pad = pad
 
+            # Monitor for events to detect packet loss/corruption
+            pad.add_probe(
+                Gst.PadProbeType.EVENT_DOWNSTREAM,
+                self.on_video_pad_probe
+            )
+
             # Check and log the original size of the video
             width = self.target_width
             height = self.target_height