#!/usr/bin/env python3
# Libervia WebRTC implementation
# Copyright (C) 2009-2026 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
from collections.abc import Awaitable
import time
import gi
gi.require_versions({"Gst": "1.0", "GstWebRTC": "1.0"})
from gi.repository import Gst, GstWebRTC, GstSdp, GLib
from libervia.backend.core import exceptions
try:
from gi.overrides import Gst as _
except ImportError:
raise exceptions.MissingModule(
"No GStreamer Python overrides available. Please install relevant packages on "
"your system (e.g., `python3-gst-1.0` on Debian and derivatives)."
)
import asyncio
from datetime import datetime
import logging
import re
from typing import Callable, Final, NamedTuple
from urllib.parse import quote_plus
from libervia.backend.tools.common import data_format
from libervia.frontends.tools import aio, display_servers, jid
from .webrtc_models import (
CallData,
SinksApp,
SinksAuto,
SinksData,
SinksDataChannel,
SinksNone,
SinksPipeline,
SourcesAuto,
SourcesData,
SourcesDataChannel,
SourcesNone,
SourcesPipeline,
SourcesStdin,
SourcesTest,
)
current_server = display_servers.detect()
if current_server == display_servers.X11:
# GSTreamer's ximagesrc documentation asks to run this function
import ctypes
ctypes.CDLL("libX11.so.6").XInitThreads()
log = logging.getLogger(__name__)
VIDEO_SOURCE_AUTO: Final = "v4l2src"
AUDIO_SOURCE_AUTO: Final = "pulsesrc"
NONE_NOT_IMPLEMENTED_MSG: Final = "None value is not handled yet."
class VideoConfig(NamedTuple):
fps: int
bitrate: int
keyframe_max: int
WEBCAM = VideoConfig(fps=24, bitrate=1200000, keyframe_max=60)
DESKTOP = VideoConfig(fps=15, bitrate=800000, keyframe_max=30)
MUTED = VideoConfig(fps=24, bitrate=100000, keyframe_max=60)
VIDEO_CONFIG = {
"video": WEBCAM,
"desktop": DESKTOP,
"muted": MUTED,
}
Gst.init(None)
class WebRTC:
"""GSTreamer based WebRTC implementation for audio and video communication.
This class encapsulates the WebRTC functionalities required for initiating and
handling audio and video calls, and data channels.
"""
def __init__(
self,
bridge,
profile: str,
sources_data: SourcesData | None = None,
sinks_data: SinksData | None = None,
reset_cb: Callable | None = None,
merge_pip: bool | None = None,
target_size: tuple[int, int] | None = None,
call_start_cb: Callable[[str, dict, str], Awaitable[str]] | None = None,
dc_data_list: list[SourcesDataChannel | SinksDataChannel] | None = None,
record_fd: int | None = None,
) -> None:
"""Initializes a new WebRTC instance.
@param bridge: An instance of backend bridge.
@param profile: Libervia profile.
@param sources_data: Data of the sources.
The model used will determine which sources to use.
SourcesDataChannel can be used here as a convenience. It will then be moved
to ``data_channels`` and ``SourcesNone`` will be used instead for
``sources_data``.
If None, SourcesAuto will be used.
@param sinks_data: Data of the sinks.
The model used will determine which sinks to use.
SinksDataChannel can be used here as a convenience. It will then be moved
to ``data_channels`` and ``SinksNone`` will be used instead for
``sinks_data``.
If None, SinksAuto will be used.
@param reset_cb: An optional Callable that is triggered on reset events. Can be
used to reset UI data on new calls.
@param merge_pip: A boolean flag indicating whether Picture-in-Picture mode is
enabled. When PiP is used, local feedback is merged to remote video stream.
Only one video stream is then produced (the local one).
If None, PiP mode is selected automatically according to selected sink (it's
used for SinksAuto only for now).
@param target_size: Expected size of the final sink stream. Mainly used by composer
when ``merge_pip`` is set.
None to autodetect (no real autodetection implemented yet, default to
(1280,720)).
@param call_start_cb: Called when call is started.
@param dc_data_list: Data Channels to create.
If a SourcesDataChannel is used as ``sources_data``, or a SinksDataChannel is
used as ``sinks_data``, they will be automatically added to this list.
@param record_fd: File descriptor for recording the incoming stream.
If None, the stream will not be recorded.
"""
self.main_loop = asyncio.get_event_loop()
self.bridge = bridge
self.profile = profile
self.pipeline = None
self._audio_muted = False
self._video_muted = False
self._desktop_sharing = False
self.desktop_sharing_data = None
if dc_data_list is None:
dc_data_list = []
self.dc_data_list = dc_data_list
if sources_data is None:
sources_data = SourcesAuto()
elif isinstance(sources_data, SourcesDataChannel):
dc_data_list.append(sources_data)
sources_data = SourcesNone()
self.sources_data = sources_data
if sinks_data is None:
sinks_data = SinksAuto()
elif isinstance(sinks_data, SinksDataChannel):
dc_data_list.append(sinks_data)
sinks_data = SinksNone()
self.sinks_data = sinks_data
if target_size is None:
target_size = (1280, 720)
self.target_width, self.target_height = target_size
if merge_pip is None:
merge_pip = isinstance(sinks_data, SinksAuto)
self.merge_pip = merge_pip
if call_start_cb is None:
call_start_cb = self._call_start
self.call_start_cb = call_start_cb
if isinstance(sinks_data, SinksApp):
if merge_pip and sinks_data.remote_video_cb is not None:
raise ValueError("Remote_video_cb can't be used when merge_pip is used!")
self.reset_cb = reset_cb
if current_server == display_servers.WAYLAND:
from .portal_desktop import DesktopPortal
self.desktop_portal = DesktopPortal(
on_session_closed_cb=self.on_portal_session_closed
)
else:
self.desktop_portal = None
self._gap_recovery_timeout = None
self._record_fd: int | None = record_fd
self._recording_elements: list[Gst.Element] = []
self._recording_muxer: Gst.Element | None = None
self.reset_instance()
@property
def audio_muted(self):
return self._audio_muted
@audio_muted.setter
def audio_muted(self, muted: bool) -> None:
if muted != self._audio_muted:
self._audio_muted = muted
self.on_audio_mute(muted)
@property
def video_muted(self):
return self._video_muted
@video_muted.setter
def video_muted(self, muted: bool) -> None:
if muted != self._video_muted:
self._video_muted = muted
self.on_video_mute(muted)
@property
def desktop_sharing(self):
return self._desktop_sharing
@desktop_sharing.setter
def desktop_sharing(self, active: bool) -> None:
if active != self._desktop_sharing:
self._desktop_sharing = active
self.on_desktop_switch(active)
try:
cb = self.bindings["desktop_sharing"]
except KeyError:
pass
else:
cb(active)
@property
def sdp_set(self):
return self._sdp_set
@sdp_set.setter
def sdp_set(self, is_set: bool):
self._sdp_set = is_set
if is_set:
self.on_ice_candidates_new(self.remote_candidates_buffer)
for data in self.remote_candidates_buffer.values():
data["candidates"].clear()
@property
def media_types(self):
if self._media_types is None:
raise Exception("self._media_types should not be None!")
return self._media_types
@media_types.setter
def media_types(self, new_media_types: dict) -> None:
self._media_types = new_media_types
self._media_types_inv = {v: k for k, v in new_media_types.items()}
@property
def media_types_inv(self) -> dict:
if self._media_types_inv is None:
raise Exception("self._media_types_inv should not be None!")
return self._media_types_inv
@property
def remote_desc_set(self) -> bool:
return self._remote_desc_set
@remote_desc_set.setter
def remote_desc_set(self, is_set: bool) -> None:
"""Setter for the remote_desc_set flag."""
if self._remote_desc_set == is_set:
return
self._remote_desc_set = is_set
if is_set:
log.debug("Remote description has been set.")
# Now that both descriptions are set, we can process buffered remote
# candidates and send our buffered local candidates.
self.on_ice_candidates_new(self.remote_candidates_buffer)
# Also send any local candidates we buffered before this point.
self._send_buffered_local_candidates()
@property
def record_fd(self) -> int | None:
return self._record_fd
@record_fd.setter
def record_fd(self, value: int | None) -> None:
"""Set or unset the recording file descriptor.
When set during an active call, starts recording.
When unset during recording, stops recording.
"""
if self._record_fd == value:
return
if value is not None:
raise NotImplementedError(
"Changing record_fd during a call is not supported yet"
)
self._record_fd = value
if value is None:
log.info("Stopping recording.")
self._remove_recording_branch()
def bind(self, **kwargs: Callable) -> None:
self.bindings.clear()
for key, cb in kwargs.items():
if key not in ("desktop_sharing",):
raise ValueError(
'Only "desktop_sharing" is currently allowed for binding'
)
self.bindings[key] = cb
def generate_dot_file(
self,
filename: str = "pipeline",
details: Gst.DebugGraphDetails = Gst.DebugGraphDetails.ALL,
with_timestamp: bool = True,
bin_: Gst.Bin | None = None,
) -> None:
"""Generate Dot File for debugging
``GST_DEBUG_DUMP_DOT_DIR`` environment variable must be set to destination dir.
``dot -Tpng -o .png .dot`` can be used to convert to a PNG
file. See
https://gstreamer.freedesktop.org/documentation/gstreamer/debugutils.html?gi-language=python#GstDebugGraphDetails
for details.
@param filename: name of the generated file
@param details: which details to print
@param with_timestamp: if True, add a timestamp to filename
@param bin_: which bin to output. By default, the whole pipeline
(``self.pipeline``) will be used.
"""
if bin_ is None:
bin_ = self.pipeline
if with_timestamp:
timestamp = datetime.now().isoformat(timespec="milliseconds")
filename = f"{timestamp}_filename"
Gst.debug_bin_to_dot_file(bin_, details, filename)
def get_sdp_mline_index(self, media_type: str) -> int:
"""Gets the sdpMLineIndex for a given media type.
@param media_type: The type of the media.
"""
for index, m_type in self.media_types.items():
if m_type == media_type:
return index
raise ValueError(f"Media type '{media_type}' not found")
def _set_media_types(self, offer_sdp: str) -> None:
"""Sets media types from offer SDP
@param offer: RTC session description containing the offer
"""
sdp_lines = offer_sdp.splitlines()
media_types = {}
mline_index = 0
for line in sdp_lines:
if line.startswith("m="):
media_types[mline_index] = line[2 : line.find(" ")]
mline_index += 1
self.media_types = media_types
log.debug(f"Media types mapping: {media_types}")
def _a_call(self, method, *args, **kwargs):
"""Call an async method in main thread"""
aio.run_from_thread(method, *args, **kwargs, loop=self.main_loop)
def get_payload_types(
self, sdpmsg, video_encoding: str, audio_encoding: str
) -> dict[str, int | None]:
"""Find the payload types for the specified video and audio encoding.
Very simplistically finds the first payload type matching the encoding
name. More complex applications will want to match caps on
profile-level-id, packetization-mode, etc.
"""
# method coming from gstreamer example (Matthew Waters, Nirbheek Chauhan) at
# subprojects/gst-examples/webrtc/sendrecv/gst/webrtc_sendrecv.py
video_pt = None
audio_pt = None
for i in range(0, sdpmsg.medias_len()):
media = sdpmsg.get_media(i)
for j in range(0, media.formats_len()):
fmt = media.get_format(j)
if fmt == "webrtc-datachannel":
continue
pt = int(fmt)
caps = media.get_caps_from_media(pt)
s = caps.get_structure(0)
encoding_name = s["encoding-name"]
if video_pt is None and encoding_name == video_encoding:
video_pt = pt
elif audio_pt is None and encoding_name == audio_encoding:
audio_pt = pt
return {video_encoding: video_pt, audio_encoding: audio_pt}
def parse_ice_candidate(self, candidate_string):
"""Parses the ice candidate string.
@param candidate_string: The ice candidate string to be parsed.
"""
# Skip empty candidates
if not candidate_string or candidate_string.strip() == "":
return None
pattern = re.compile(
r"candidate:(?P\S+) (?P\d+) (?P\S+) "
r"(?P\d+) (?P\S+) (?P\d+) typ "
r"(?P\S+)(?: raddr (?P\S+) rport "
r"(?P\d+))?(?: generation (?P\d+))?"
)
match = pattern.match(candidate_string)
if match:
candidate_dict = match.groupdict()
# Apply the correct types to the dictionary values
candidate_dict["component_id"] = int(candidate_dict["component_id"])
candidate_dict["priority"] = int(candidate_dict["priority"])
candidate_dict["port"] = int(candidate_dict["port"])
if candidate_dict["rel_port"]:
candidate_dict["rel_port"] = int(candidate_dict["rel_port"])
if candidate_dict["generation"]:
candidate_dict["generation"] = candidate_dict["generation"]
# Remove None values
return {k: v for k, v in candidate_dict.items() if v is not None}
else:
log.warning(f"can't parse candidate: {candidate_string!r}")
return None
def build_ice_candidate(self, parsed_candidate):
"""Builds ICE candidate
@param parsed_candidate: Dictionary containing parsed ICE candidate
"""
base_format = (
"candidate:{foundation} {component_id} {transport} {priority} "
"{address} {port} typ {type}"
)
if parsed_candidate.get("rel_addr") and parsed_candidate.get("rel_port"):
base_format += " raddr {rel_addr} rport {rel_port}"
if parsed_candidate.get("generation"):
base_format += " generation {generation}"
return base_format.format(**parsed_candidate)
def extract_ufrag_pwd(self, sdp: str) -> None:
"""Retrieves ICE password and user fragment for SDP offer.
@param sdp: The Session Description Protocol offer string.
"""
lines = sdp.splitlines()
media = ""
mid_media_map = {}
bundle_media = set()
bundle_ufrag = ""
bundle_pwd = ""
in_bundle = False
for line in lines:
if line.startswith("m="):
media = line.split("=")[1].split()[0]
elif line.startswith("a=mid:"):
mid = line.split(":")[1].strip()
mid_media_map[mid] = media
elif line.startswith("a=group:BUNDLE"):
in_bundle = True
bundle_media = set(line.split(":")[1].strip().split())
elif line.startswith("a=ice-ufrag:"):
if in_bundle:
bundle_ufrag = line.split(":")[1].strip()
else:
self.ufrag[media] = line.split(":")[1].strip()
elif line.startswith("a=ice-pwd:"):
if in_bundle:
bundle_pwd = line.split(":")[1].strip()
else:
self.pwd[media] = line.split(":")[1].strip()
else:
in_bundle = False
if bundle_ufrag and bundle_pwd:
for mid in bundle_media:
media = mid_media_map.get(mid)
if media:
self.ufrag[media] = bundle_ufrag
self.pwd[media] = bundle_pwd
log.debug(
f"Set bundle ICE credentials for {media}: ufrag={bundle_ufrag[:4]}..."
)
else:
log.warning(f"No media type found for mid {mid}")
def reset_instance(self):
"""Inits or resets the instance variables to their default state."""
if self._recording_elements:
self._remove_recording_branch()
self.role: str | None = None
if self.pipeline is not None:
self.set_pipeline_state(Gst.State.NULL)
self.pipeline = None
self._remote_video_pad = None
self.sid: str | None = None
self.offer: str | None = None
self.local_candidates_buffer = {}
self.ufrag: dict[str, str] = {}
self.pwd: dict[str, str] = {}
self.callee: jid.JID | None = None
self._media_types = None
self._media_types_inv = None
self._sdp_set: bool = False
self._remote_desc_set: bool = False
self.remote_candidates_buffer: dict[str, dict[str, list]] = {
"audio": {"candidates": []},
"video": {"candidates": []},
}
self._media_types = None
self._media_types_inv = None
self.audio_valve = None
self.video_valve = None
self.video_selector = None
if self.desktop_portal is not None:
self.desktop_portal.end_session()
self.desktop_sharing = False
self.desktop_sink_pad = None
self.bindings = {}
if self.reset_cb is not None:
self.reset_cb()
self.data_channels: dict[str, GstWebRTC.WebRTCDataChannel] = {}
if self._gap_recovery_timeout is not None:
GLib.source_remove(self._gap_recovery_timeout)
self._gap_recovery_timeout = None
@property
def data_channel(self) -> GstWebRTC.WebRTCDataChannel:
"""Convenience method to get WebRTCDataChannel instance when there is only one."""
if len(self.data_channels) != 1:
raise exceptions.InternalError(
"self.data_channel can only be used in a single Data Channel scenario. "
"Use self.data_channels dict instead."
)
return next(iter(self.data_channels.values()))
async def setup_call(
self,
role: str,
audio_pt: int | None = 96,
video_pt: int | None = 97,
) -> None:
"""Sets up the call.
This method establishes the Gstreamer pipeline for audio and video communication.
The method also manages STUN and TURN server configurations, signal watchers, and
various connection handlers for the webrtcbin.
@param role: The role for the call, either 'initiator' or 'responder'.
@param audio_pt: The payload type for the audio stream.
@param video_pt: The payload type for the video stream
@raises NotImplementedError: If audio_pt or video_pt is set to None.
@raises AssertionError: If the role is not 'initiator' or 'responder'.
"""
assert role in ("initiator", "responder")
self.role = role
if isinstance(self.sources_data, SourcesPipeline):
if self.sources_data.video_pipeline != "" and video_pt is None:
raise NotImplementedError(NONE_NOT_IMPLEMENTED_MSG)
if self.sources_data.audio_pipeline != "" and audio_pt is None:
raise NotImplementedError(NONE_NOT_IMPLEMENTED_MSG)
elif isinstance(self.sources_data, SourcesNone):
pass
elif isinstance(self.sources_data, SourcesStdin):
# SourcesStdin uses decodebin which will dynamically provide streams.
if audio_pt is None or video_pt is None:
raise NotImplementedError(NONE_NOT_IMPLEMENTED_MSG)
else:
if audio_pt is None or video_pt is None:
raise NotImplementedError(NONE_NOT_IMPLEMENTED_MSG)
match self.sources_data:
case SourcesAuto():
video_source_elt = VIDEO_SOURCE_AUTO
audio_source_elt = AUDIO_SOURCE_AUTO
case SourcesNone():
video_source_elt = ""
audio_source_elt = ""
case SourcesPipeline() as source:
if source.video_pipeline is None:
video_source_elt = VIDEO_SOURCE_AUTO
else:
video_source_elt = source.video_pipeline
if source.audio_pipeline is None:
audio_source_elt = AUDIO_SOURCE_AUTO
else:
audio_source_elt = source.audio_pipeline
case SourcesTest():
video_source_elt = "videotestsrc is-live=true pattern=ball"
audio_source_elt = "audiotestsrc"
case SourcesStdin():
video_source_elt = ""
audio_source_elt = ""
case _:
raise exceptions.InternalError(
f'Unexpected "sources_data" value: {self.sources_data!r}'
)
match self.sinks_data:
case SinksApp():
local_video_sink_elt = (
"appsink name=local_video_sink emit-signals=true drop=true "
"max-buffers=2 sync=false"
)
case SinksAuto():
if isinstance(self.sources_data, SourcesNone):
local_video_sink_elt = ""
else:
local_video_sink_elt = "autovideosink"
case SinksNone() | SinksPipeline():
local_video_sink_elt = ""
case _:
raise exceptions.InternalError(
f'Unexpected "sinks_data" value {self.sinks_data!r}'
)
gst_pipe_elements = [
"webrtcbin latency=200 name=sendrecv bundle-policy=max-bundle"
]
if self.merge_pip and local_video_sink_elt:
# Compositor is used to merge local video feedback in video sink, useful when
# we have only a single video sink.
gst_pipe_elements.append(
"compositor name=compositor background=black "
f"! video/x-raw,width={self.target_width},"
f"height={self.target_height},framerate={WEBCAM.fps}/1 "
f"! {local_video_sink_elt}"
)
local_video_sink_elt = "compositor.sink_1"
if video_source_elt or isinstance(self.sources_data, SourcesStdin):
# Video source with an input-selector to switch between normal and video mute
# (or desktop sharing).
video_elements = [
"input-selector name=video_selector",
"! tee name=t",
"",
]
if video_source_elt:
video_elements.extend(
[
f"{video_source_elt} name=video_src",
"! queue max-size-buffers=3 leaky=downstream",
"! video_selector.",
"",
]
)
video_elements.extend(
[
"videotestsrc name=muted_src is-live=true pattern=black",
"! queue leaky=downstream",
"! video_selector.",
"",
"t.",
"! queue max-size-buffers=3 leaky=downstream",
"! videorate",
f"! capsfilter name=main_rate_caps caps=video/x-raw,framerate={WEBCAM.fps}/1",
"! videoconvert",
f"! vp8enc name=vp8enc deadline=1 keyframe-max-dist={WEBCAM.keyframe_max} cpu-used=5 target-bitrate={WEBCAM.bitrate} error-resilient=1",
"! rtpvp8pay picture-id-mode=15-bit mtu=1200",
f"! application/x-rtp,media=video,encoding-name=VP8,payload={video_pt}",
"! sendrecv.",
]
)
gst_pipe_elements.append("\n".join(video_elements))
if local_video_sink_elt and (
video_source_elt or isinstance(self.sources_data, SourcesStdin)
):
# Local video feedback.
gst_pipe_elements.append(f"""
t.
! queue max-size-buffers=3 leaky=downstream
! videoconvert
! {local_video_sink_elt}
""")
if audio_source_elt or isinstance(self.sources_data, SourcesStdin):
# Audio with a valve for muting.
audio_elements = []
if audio_source_elt:
audio_elements.extend(
[
f"{audio_source_elt} name=audio_src",
"! valve name=audio_valve",
]
)
else:
# For stdin, just create the valve for dynamic linking
audio_elements.append("valve name=audio_valve")
audio_elements.extend(
[
"! queue max-size-buffers=10 max-size-time=0 max-size-bytes=0 leaky=downstream",
"! audioconvert",
"! audioresample",
"! opusenc audio-type=voice bitrate=32000",
"! rtpopuspay",
f"! application/x-rtp,media=audio,encoding-name=OPUS,payload={audio_pt}",
"! sendrecv.",
]
)
gst_pipe_elements.append("\n".join(audio_elements))
self.gst_pipe_desc = "\n\n".join(gst_pipe_elements)
log.debug(f"Gstreamer pipeline: {self.gst_pipe_desc}")
# Create the pipeline
try:
self.pipeline = Gst.parse_launch(self.gst_pipe_desc)
except Exception:
log.exception("Can't parse pipeline")
self.pipeline = None
if not self.pipeline:
raise exceptions.InternalError("Failed to create Gstreamer pipeline.")
if not isinstance(self.pipeline, Gst.Pipeline):
# in the case of Data Channel there is a single element, and Gst.parse_launch
# doesn't create a Pipeline in this case, so we do it manually.
pipeline = Gst.Pipeline()
pipeline.add(self.pipeline)
self.pipeline = pipeline
self.webrtcbin = self.pipeline.get_by_name("sendrecv")
if self.webrtcbin is None:
raise exceptions.InternalError("Can't get the pipeline.")
self.monitor_connection_state()
# If video or audio sources are not created, ``get_by_name`` will return None.
self.video_src = self.pipeline.get_by_name("video_src")
self.muted_src = self.pipeline.get_by_name("muted_src")
self.video_selector = self.pipeline.get_by_name("video_selector")
if self.video_src and isinstance(self.sources_data, SourcesPipeline):
for name, value in self.sources_data.video_properties.items():
self.video_src.set_property(name, value)
self.audio_src = self.pipeline.get_by_name("audio_src")
if self.audio_src and isinstance(self.sources_data, SourcesPipeline):
for name, value in self.sources_data.audio_properties.items():
self.audio_src.set_property(name, value)
self.audio_valve = self.pipeline.get_by_name("audio_valve")
if self.video_muted:
self.on_video_mute(True)
if self.audio_muted:
self.on_audio_mute(True)
# set STUN and TURN servers
external_disco = data_format.deserialise(
await self.bridge.external_disco_get("", self.profile), type_check=list
)
for server in external_disco:
if server["type"] == "stun":
if server["transport"] == "tcp":
log.info(
"ignoring TCP STUN server, GStreamer only support one STUN server"
)
url = f"stun://{server['host']}:{server['port']}"
log.debug(f"adding stun server: {url}")
self.webrtcbin.set_property("stun-server", url)
elif server["type"] == "turn":
url = "{scheme}://{username}:{password}@{host}:{port}".format(
scheme="turns" if server["transport"] == "tcp" else "turn",
username=quote_plus(server["username"]),
password=quote_plus(server["password"]),
host=server["host"],
port=server["port"],
)
log.debug(f"adding turn server: {url}")
if not self.webrtcbin.emit("add-turn-server", url):
log.warning(f"Erreur while adding TURN server {url}")
# local video feedback
if isinstance(self.sinks_data, SinksApp):
local_video_sink = self.pipeline.get_by_name("local_video_sink")
if local_video_sink is not None:
local_video_sink.set_property("emit-signals", True)
local_video_sink.connect("new-sample", self.sinks_data.local_video_cb)
local_video_sink_caps = Gst.Caps.from_string(f"video/x-raw,format=RGB")
local_video_sink.set_property("caps", local_video_sink_caps)
if isinstance(self.sources_data, SourcesStdin):
self._setup_stdin_source()
# Create bus and associate signal watchers
self.bus = self.pipeline.get_bus()
if not self.bus:
log.error("Failed to get bus from pipeline.")
return
self.bus.add_signal_watch()
self.webrtcbin.connect("pad-added", self.on_pad_added)
self.bus.connect("message::error", self.on_bus_error)
self.bus.connect("message::eos", self.on_bus_eos)
self.webrtcbin.connect("on-negotiation-needed", self.on_negotiation_needed)
self.webrtcbin.connect("on-ice-candidate", self.on_ice_candidate)
self.webrtcbin.connect(
"notify::ice-gathering-state", self.on_ice_gathering_state_change
)
self.webrtcbin.connect(
"notify::ice-connection-state", self.on_ice_connection_state
)
for dc_data in self.dc_data_list:
self.create_data_channel(dc_data)
def create_data_channel(self, dc_data: SourcesDataChannel | SinksDataChannel) -> None:
"""Create a Data Channel and connect relevant callbacks."""
assert self.pipeline is not None
match dc_data:
case SourcesDataChannel():
# Data channel configuration for compatibility with browser defaults
data_channel_options = Gst.Structure.new_empty("data-channel-options")
data_channel_options.set_value("ordered", True)
data_channel_options.set_value("protocol", "")
# Create the data channel
self.pipeline.set_state(Gst.State.READY)
self.data_channels[dc_data.name] = data_channel = self.webrtcbin.emit(
"create-data-channel", dc_data.name, data_channel_options
)
if data_channel is None:
log.error("Failed to create data channel")
return
data_channel.connect("on-open", dc_data.dc_open_cb)
case SinksDataChannel():
self.webrtcbin.connect("on-data-channel", dc_data.dc_on_data_channel)
case _:
raise ValueError(
"Only SourcesDataChannel or SinksDataChannel are allowed."
)
def set_pipeline_state(self, state: Gst.State) -> bool:
"""Safely set pipeline state with verification.
@param state: The target state
@return: True if state change was successful
"""
if not self.pipeline:
log.error("Pipeline not initialized")
return False
ret = self.pipeline.set_state(state)
if ret == Gst.StateChangeReturn.FAILURE:
log.error(f"Failed to change pipeline state to {state}")
return False
elif ret == Gst.StateChangeReturn.ASYNC:
log.debug(f"Waiting for async state change to {state.value_nick.upper()}...")
ret, _, _ = self.pipeline.get_state(timeout=5 * Gst.SECOND)
if ret != Gst.StateChangeReturn.SUCCESS:
log.warning(
f"Async state change to {state.value_nick.upper()} timed out or failed!"
)
return False
log.debug(f"Pipeline state successfully changed to {state.value_nick.upper()}")
return True
def monitor_connection_state(self):
"""Monitor WebRTC connection state"""
def on_notify_connection_state(pspect, param):
state = self.webrtcbin.get_property("connection-state")
log.debug(f"WebRTC connection state: {state}")
if state == GstWebRTC.WebRTCPeerConnectionState.FAILED:
log.error("WebRTC connection failed")
elif state == GstWebRTC.WebRTCPeerConnectionState.CONNECTED:
log.info("WebRTC connection established")
self.webrtcbin.connect("notify::connection-state", on_notify_connection_state)
def start_pipeline(self) -> None:
"""Starts the GStreamer pipeline."""
log.debug("starting the pipeline")
self.set_pipeline_state(Gst.State.PLAYING)
def on_negotiation_needed(self, webrtc):
"""Initiate SDP offer when negotiation is needed."""
log.debug("Negotiation needed.")
if self.role == "initiator":
log.debug("Creating offer…")
promise = Gst.Promise.new_with_change_func(self.on_offer_created)
self.webrtcbin.emit("create-offer", None, promise)
def on_offer_created(self, promise):
"""Callback for when SDP offer is created."""
log.debug("on_offer_created called")
assert promise.wait() == Gst.PromiseResult.REPLIED
reply = promise.get_reply()
if reply is None:
log.error("Promise reply is None. Offer creation might have failed.")
return
offer = reply["offer"]
self.offer = offer.sdp.as_text()
log.debug(f"SDP offer created: \n{self.offer}")
self._set_media_types(self.offer)
promise = Gst.Promise.new()
self.webrtcbin.emit("set-local-description", offer, promise)
promise.interrupt()
self._a_call(self._start_call)
def on_answer_set(self, promise):
assert promise.wait() == Gst.PromiseResult.REPLIED
def on_answer_created(self, promise, _, __):
"""Callback for when SDP answer is created."""
assert promise.wait() == Gst.PromiseResult.REPLIED
reply = promise.get_reply()
answer = reply["answer"]
promise = Gst.Promise.new()
self.webrtcbin.emit("set-local-description", answer, promise)
promise.interrupt()
answer_sdp = answer.sdp.as_text()
log.debug(f"SDP answer set: \n{answer_sdp}")
self.sdp_set = True
self._a_call(self.bridge.call_answer_sdp, self.sid, answer_sdp, self.profile)
def on_offer_set(self, promise):
assert promise.wait() == Gst.PromiseResult.REPLIED
promise = Gst.Promise.new_with_change_func(self.on_answer_created, None, None)
self.webrtcbin.emit("create-answer", None, promise)
def link_element_or_pad(
self, source: Gst.Element, dest: Gst.Element | Gst.Pad
) -> bool:
"""Check if dest is a pad or an element, and link appropriately"""
src_pad = source.get_static_pad("src")
if isinstance(dest, Gst.Pad):
# If the dest is a pad, link directly
if not src_pad.link(dest) == Gst.PadLinkReturn.OK:
log.error(
"Failed to link 'conv' to the compositor's newly requested pad!"
)
return False
elif isinstance(dest, Gst.Element):
if not source.link(dest):
log.error(f"Failed to link {source.get_name()} to {dest.get_name()}")
return False
else:
log.error(f"Unexpected type for dest: {type(dest)}")
return False
return True
def scaled_dimensions(
self, original_width: int, original_height: int, max_width: int, max_height: int
) -> tuple[int, int]:
"""Calculates the scaled dimensions preserving aspect ratio.
@param original_width: Original width of the video stream.
@param original_height: Original height of the video stream.
@param max_width: Maximum desired width for the scaled video.
@param max_height: Maximum desired height for the scaled video.
@return: The width and height of the scaled video.
"""
aspect_ratio = original_width / original_height
new_width = int(max_height * aspect_ratio)
if new_width <= max_width:
return new_width, max_height
new_height = int(max_width / aspect_ratio)
return max_width, new_height
def on_video_pad_probe(
self, pad: Gst.Pad, info: Gst.PadProbeInfo
) -> Gst.PadProbeReturn:
"""Monitor video pad for GAP events indicating packet loss.
When gaps are detected, mark that we need recovery. After gaps stop,
wait a bit then request a keyframe.
"""
event = info.get_event()
if event and event.type == Gst.EventType.GAP:
log.debug("Video packet loss detected (GAP event).")
if self._gap_recovery_timeout is not None:
GLib.source_remove(self._gap_recovery_timeout)
self._gap_recovery_timeout = None
# Schedule keyframe request after gap stops.
# Wait 1 second after last GAP event to ensure connection has recovered
self._gap_recovery_timeout = GLib.timeout_add_seconds(
1, self._on_gap_recovery
)
return Gst.PadProbeReturn.OK
def _on_gap_recovery(self) -> bool:
"""Called after GAP events have stopped, indicating connection recovery.
@return: False to remove the timeout (one-shot)
"""
log.info("Connection recovering after packet loss, requesting keyframe.")
self._request_keyframe()
self._gap_recovery_timeout = None
# We return False to make it a one-short timer callback.
return False
def _request_keyframe(self) -> None:
"""Requests a keyframe from the remote peer.
A keyframe is requested by sending a GstForceKeyUnit event upstream. This is used
for video recovery after packet loss.
"""
if not self._remote_video_pad:
log.warning("Cannot request keyframe, remote video pad not yet available.")
return
decodebin = self._remote_video_pad.get_parent()
if not decodebin or not isinstance(decodebin, Gst.Element):
log.warning("Could not get parent element (decodebin) from remote video pad.")
return
log.debug("Requesting keyframe from remote peer (sending PLI/FIR).")
# Create the GstForceKeyUnit event. This event, when it reaches the
# rtpjitterbuffer inside webrtcbin, will cause it to send an
# RTCP Picture Loss Indication (PLI) or Full Intra Request (FIR)
# packet to the sender.
structure = Gst.Structure.new_from_string("GstForceKeyUnit")
structure.set_value("all-headers", True)
event = Gst.Event.new_custom(Gst.EventType.CUSTOM_UPSTREAM, structure)
if decodebin.send_event(event):
log.debug("Successfully sent GstForceKeyUnit event to decodebin.")
else:
log.error("Failed to send GstForceKeyUnit event to decodebin.")
def _remove_recording_branch(self) -> None:
"""Removes the recording branch from the pipeline."""
if not self.pipeline or not self._recording_elements:
log.warning("No recording branch to remove")
return
log.info("Removing recording branch from the pipeline.")
current_state = self.pipeline.get_state(0)[1]
if current_state == Gst.State.PLAYING:
self.set_pipeline_state(Gst.State.PAUSED)
try:
# Release muxer pads before removing elements
if self._recording_muxer:
for element in self._recording_elements:
if "depay" in element.get_name():
depay_src_pad = element.get_static_pad("src")
if depay_src_pad:
peer_pad = depay_src_pad.get_peer()
if (
peer_pad
and self._recording_muxer == peer_pad.get_parent()
):
log.debug(f"Releasing muxer pad: {peer_pad.get_name()}")
self._recording_muxer.release_request_pad(peer_pad)
# Remove elements in reverse order
elements_to_remove = list(self._recording_elements)
for element in reversed(elements_to_remove):
if element.get_parent() == self.pipeline:
log.debug(f"Removing recording element: {element.get_name()}")
element.set_state(Gst.State.NULL)
self.pipeline.remove(element)
self._recording_elements.remove(element)
self._recording_muxer = None
log.info("Recording branch successfully removed")
except Exception:
log.exception("Error removing recording branch")
self._recording_elements.clear()
self._recording_muxer = None
finally:
if current_state == Gst.State.PLAYING:
self.set_pipeline_state(Gst.State.PLAYING)
def on_remote_decodebin_stream(self, decodebin, pad: Gst.Pad) -> None:
"""Handle the stream from the remote decodebin.
This method processes the incoming stream from the remote decodebin, determining
whether it's video or audio. It then sets up the appropriate GStreamer elements
for video/audio processing and adds them to the pipeline.
@param decodebin: signal source.
@param pad: The Gst.Pad from the remote decodebin producing the stream.
"""
assert self.pipeline is not None
caps = pad.get_current_caps()
if not caps:
caps = pad.query_caps(None)
name = caps.get_structure(0).get_name()
log.debug(f"Handling decoded pad of type {name}")
queue = Gst.ElementFactory.make("queue")
if name.startswith("video/"):
log.debug("===> VIDEO OK")
self._remote_video_pad = pad
# Monitor for events to detect packet loss/corruption
pad.add_probe(Gst.PadProbeType.EVENT_DOWNSTREAM, self.on_video_pad_probe)
# Check and log the original size of the video
structure = caps.get_structure(0)
width = (
structure.get_int("width")[1]
if structure.has_field("width")
else self.target_width
)
height = (
structure.get_int("height")[1]
if structure.has_field("height")
else self.target_height
)
log.debug(f"Original video size: {width}x{height}")
log.debug(f"Target size: {self.target_width}x{self.target_height}")
# This is a fix for an issue found with Movim on desktop: a non standard
# resolution is used (990x557) resulting in bad alignement and no color in
# rendered image
adjust_resolution = width % 4 != 0 or height % 4 != 0
if adjust_resolution:
log.warning("non standard resolution, we need to adjust size")
width = (width + 3) // 4 * 4
height = (height + 3) // 4 * 4
log.info(f"Adjusted video size: {width}x{height}")
convert = Gst.ElementFactory.make("videoconvert")
if self.merge_pip:
# with ``merge_pip`` set, we plug the remote stream to the composer
compositor = self.pipeline.get_by_name("compositor")
sink1_pad = compositor.get_static_pad("sink_1")
local_width, local_height = self.scaled_dimensions(
self.target_width,
self.target_height,
width // 3,
height // 3,
)
sink1_pad.set_property("xpos", self.target_width - local_width)
sink1_pad.set_property("ypos", self.target_height - local_height)
sink1_pad.set_property("width", local_width)
sink1_pad.set_property("height", local_height)
sink1_pad.set_property("sizing-policy", 1)
sink1_pad.set_property("zorder", 1)
# Request a new pad for the remote stream
sink_pad_template = compositor.get_pad_template("sink_%u")
remote_video_sink = compositor.request_pad(sink_pad_template, None, None)
remote_video_sink.set_property("zorder", 0)
remote_video_sink.set_property("width", self.target_width)
remote_video_sink.set_property("height", self.target_height)
remote_video_sink.set_property("sizing-policy", 1)
elif isinstance(self.sinks_data, SinksApp):
# ``app`` sink without ``self.merge_pip`` set, be create the sink and
# connect it to the ``remote_video_cb``.
remote_video_sink = Gst.ElementFactory.make(
"appsink", "remote_video_sink"
)
remote_video_sink.set_property(
"caps", Gst.Caps.from_string("video/x-raw,format=RGB")
)
remote_video_sink.set_property("emit-signals", True)
remote_video_sink.set_property("drop", True)
remote_video_sink.set_property("max-buffers", 10)
remote_video_sink.set_property("max-time", 500000000)
remote_video_sink.set_property("processing-deadline", 50000000)
remote_video_sink.set_property("sync", False)
remote_video_sink.set_property("qos", True)
if self.sinks_data.remote_video_cb:
remote_video_sink.connect(
"new-sample", self.sinks_data.remote_video_cb
)
else:
log.error(
"sinks_data.remote_video_cb is not available! Cannot display remote video."
)
return
elif isinstance(self.sinks_data, SinksAuto) or (
# We have custom sinks, but video is automatic.
isinstance(self.sinks_data, SinksPipeline)
and self.sinks_data.video_pipeline is None
):
# if ``self.merge_pip`` is not set, we create a dedicated
# ``autovideosink`` for remote stream.
remote_video_sink = Gst.ElementFactory.make("autovideosink")
elif isinstance(self.sinks_data, SinksPipeline):
# We have a custom video pipeline.
if self.sinks_data.video_pipeline == "":
# Video sink is deactivated.
remote_video_sink = Gst.ElementFactory.make("fakesink")
else:
remote_video_sink = Gst.parse_bin_from_description(
self.sinks_data.video_pipeline
)
iterator = remote_video_sink.iterate_elements()
ret, first_element = iterator.next()
if ret == Gst.IteratorResult.OK:
for name, value in self.sinks_data.video_properties.items():
first_element.set_property(name, value)
else:
log.warning("No elements found in the bin.")
else:
raise exceptions.InternalError(
f'Unhandled "sinks_data" value: {self.sinks_data!r}'
)
if adjust_resolution:
log.debug("Applying adjust resolution pipeline.")
videoscale = Gst.ElementFactory.make("videoscale")
adjusted_caps = Gst.Caps.from_string(
f"video/x-raw,width={width},height={height}"
)
capsfilter = Gst.ElementFactory.make("capsfilter")
capsfilter.set_property("caps", adjusted_caps)
# Add elements to pipeline
if self.merge_pip:
self.pipeline.add(queue, convert, videoscale, capsfilter)
else:
self.pipeline.add(
queue, convert, videoscale, capsfilter, remote_video_sink
)
ret = pad.link(queue.get_static_pad("sink"))
if ret != Gst.PadLinkReturn.OK:
log.error(f"Error linking pad to queue: {ret}")
return
if not Gst.Element.link_many(queue, convert, videoscale, capsfilter):
log.error("Failed to link adjustment pipeline chain")
return
if not self.link_element_or_pad(capsfilter, remote_video_sink):
log.error("Failed to link capsfilter to sink")
return
queue.sync_state_with_parent()
convert.sync_state_with_parent()
videoscale.sync_state_with_parent()
capsfilter.sync_state_with_parent()
if not self.merge_pip:
remote_video_sink.sync_state_with_parent()
else:
if self.merge_pip:
self.pipeline.add(queue, convert)
else:
self.pipeline.add(queue, convert, remote_video_sink)
ret = pad.link(queue.get_static_pad("sink"))
if ret != Gst.PadLinkReturn.OK:
log.error(f"Error linking pad to queue: {ret}")
return
if not queue.link(convert):
log.error("Failed to link queue to convert")
return
if not self.link_element_or_pad(convert, remote_video_sink):
log.error("Failed to link convert to sink")
return
queue.sync_state_with_parent()
convert.sync_state_with_parent()
if not self.merge_pip:
remote_video_sink.sync_state_with_parent()
log.info("Successfully linked remote video stream.")
elif name.startswith("audio/"):
log.debug("===> Audio OK")
# Create the audio sink chain: queue -> convert -> resample -> sink
if isinstance(self.sinks_data, SinksPipeline):
# We have a custom audio pipeline.
match self.sinks_data.audio_pipeline:
case None:
# Audio sink is automatic.
remote_audio_sink = Gst.ElementFactory.make("autoaudiosink")
case "":
# Audio sink is deactivated.
remote_audio_sink = Gst.ElementFactory.make("fakesink")
self.pipeline.add(queue, remote_audio_sink)
pad.link(queue.get_static_pad("sink"))
queue.link(remote_audio_sink)
queue.sync_state_with_parent()
remote_audio_sink.sync_state_with_parent()
log.info("Audio stream won't be played live.")
return
case _:
remote_audio_sink = Gst.parse_bin_from_description(
self.sinks_data.audio_pipeline
)
iterator = remote_audio_sink.iterate_elements()
ret, first_element = iterator.next()
if ret == Gst.IteratorResult.OK:
for name, value in self.sinks_data.audio_properties.items():
first_element.set_property(name, value)
else:
log.warning("No elements found in the bin.")
else:
remote_audio_sink = Gst.ElementFactory.make("autoaudiosink")
convert = Gst.ElementFactory.make("audioconvert")
resample = Gst.ElementFactory.make("audioresample")
self.pipeline.add(queue, convert, resample, remote_audio_sink)
pad.link(queue.get_static_pad("sink"))
Gst.Element.link_many(queue, convert, resample, remote_audio_sink)
# Bring all new elements to the PLAYING state
queue.sync_state_with_parent()
convert.sync_state_with_parent()
resample.sync_state_with_parent()
remote_audio_sink.sync_state_with_parent()
log.info("Successfully linked remote audio stream.")
else:
log.warning(f"Ignoring unknown pad type from decodebin: {name}")
def on_pad_added(self, webrtcbin, pad: Gst.Pad) -> None:
"""Handle the addition of a new pad from webrtcbin.
Always creates a tee to allow dynamic recording branch addition.
The tee splits the stream: one branch goes to decodebin for playback,
another can go to recording if enabled.
@param webrtcbin: Signal source.
@param pad: The newly added pad.
"""
assert self.pipeline is not None
log.debug(f"Handling new source pad {pad.get_name()}.")
if pad.direction != Gst.PadDirection.SRC:
log.debug(f"Ignoring non-source pad: {pad.get_name()}")
return
# Determine media type from caps.
caps = pad.get_current_caps() or pad.query_caps(None)
if not caps or caps.get_size() == 0:
log.warning("Empty caps on pad, ignoring.")
return
structure = caps.get_structure(0)
structure_name = structure.get_name()
log.debug(f"Pad caps structure: {structure_name}.")
# We only handle RTP stream.
if structure_name != "application/x-rtp":
log.debug(f"Ignoring non-RTP pad with caps: {structure_name}.")
return
# Extract media type.
media_type = structure.get_string("media")
if media_type not in ["video", "audio"]:
log.warning(f"Ignoring RTP pad with unexpected media type: {media_type}.")
return
log.debug(f"Processing incoming {media_type} stream.")
current_state = self.pipeline.get_state(0)[1]
needs_state_restore = current_state == Gst.State.PLAYING
if needs_state_restore:
log.debug("Pausing pipeline for safe modification.")
self.set_pipeline_state(Gst.State.PAUSED)
tee_name = f"{media_type}_tee"
decodebin_name = f"decodebin_{media_type}"
try:
# Create tee for this media stream (always present)
tee = Gst.ElementFactory.make("tee", tee_name)
if not tee:
log.error(f"Failed to create tee for {media_type}.")
return
self.pipeline.add(tee)
# Link: webrtcbin -> tee
if pad.link(tee.get_static_pad("sink")) != Gst.PadLinkReturn.OK:
log.error(f"Failed to link webrtcbin pad to tee for {media_type}.")
self.pipeline.remove(tee)
return
tee.sync_state_with_parent()
if (
True
# FIXME: If the playback branch is not run, only audio stream is recorded.
# Something is missing when recording branch only is used. Requesting
# keyframe doesn't solve the issue. So for now we always activate
# playback branch, until the problem is fixed. To deactivate playback
# branch when no UI is shown, just uncomment the 3 next lines.
# not isinstance(self.sinks_data, SinksPipeline)
# or self.sinks_data.audio_pipeline != ""
# or self.sinks_data.video_pipeline != ""
):
# There is a playback branch.
# Create decodebin for playback.
decodebin = Gst.ElementFactory.make("decodebin", decodebin_name)
if not decodebin:
log.error(f"Failed to create decodebin for {media_type}.")
self.pipeline.remove(tee)
return
decodebin.connect("pad-added", self.on_remote_decodebin_stream)
self.pipeline.add(decodebin)
# Link tee -> decodebin (playback branch)
playback_pad = tee.get_request_pad("src_%u")
if (
playback_pad.link(decodebin.get_static_pad("sink"))
!= Gst.PadLinkReturn.OK
):
log.error(f"Failed to link tee to decodebin for {media_type}.")
self.pipeline.remove(tee)
self.pipeline.remove(decodebin)
return
decodebin.sync_state_with_parent()
# If recording is active, add recording branch from tee
if self._record_fd is not None:
log.info(
f"Recording is active, adding recording branch for {media_type}."
)
recording_pad = tee.get_request_pad("src_%u")
if not self._add_rtp_recording_branch(recording_pad, media_type):
log.warning(f"Failed to add recording branch for {media_type}")
tee.release_request_pad(recording_pad)
else:
log.info(f"Recording branch added for {media_type}.")
log.info(f"Successfully set up {media_type} pipeline with tee.")
except Exception:
log.exception(f"Error in on_pad_added for {media_type}.")
self._cleanup_elements([tee_name, decodebin_name])
finally:
if needs_state_restore:
log.debug("Restoring pipeline to PLAYING state.")
self.set_pipeline_state(Gst.State.PLAYING)
log.debug("Finished handling pad addition")
def _add_rtp_recording_branch(self, src_pad: Gst.Pad, media_type: str) -> bool:
"""Add recording branch for RTP stream.
RTP stream are muxed and written as-is, without re-encoding.
@param src_pad: Source pad.
@param media_type: Type of the media ("video" or "audio").
@return: True is successful.
"""
assert self.pipeline is not None
assert self._record_fd is not None
log.debug(f"Adding RTP recording branch for {media_type}")
queue = Gst.ElementFactory.make("queue", f"rec_queue_{media_type}")
if not queue:
log.error(f"Failed to create queue for {media_type} recording")
return False
depayloader_type = "rtpvp8depay" if media_type == "video" else "rtpopusdepay"
depayloader_name = f"{depayloader_type}_rec_{media_type}"
depayloader = Gst.ElementFactory.make(depayloader_type, depayloader_name)
if not depayloader:
log.error(f"Failed to create {depayloader_type}")
return False
# Create muxer and sink if they don't exist
if self._recording_muxer is None:
self._recording_muxer = Gst.ElementFactory.make("webmmux", "recording_muxer")
fdsink = Gst.ElementFactory.make("fdsink", "recording_sink")
if not self._recording_muxer or not fdsink:
log.error("Failed to create muxer or sink")
self._recording_muxer = None
return False
# Configure muxer for live streaming
self._recording_muxer.set_property("streamable", True)
fdsink.set_property("fd", self._record_fd)
fdsink.set_property("sync", False)
fdsink.set_property("async", False)
self.pipeline.add(self._recording_muxer)
self.pipeline.add(fdsink)
if not self._recording_muxer.link(fdsink):
log.error("Failed to link muxer to sink")
self._cleanup_elements(
[self._recording_muxer, fdsink], self._recording_elements
)
self._recording_muxer = None
return False
self._recording_elements.extend([self._recording_muxer, fdsink])
self._recording_muxer.sync_state_with_parent()
fdsink.sync_state_with_parent()
self.pipeline.add(queue)
self.pipeline.add(depayloader)
# Link: src_pad -> queue -> depayloader
if src_pad.link(queue.get_static_pad("sink")) != Gst.PadLinkReturn.OK:
log.error(f"Failed to link tee src pad to queue for {media_type}")
self._cleanup_elements([queue, depayloader], self._recording_elements)
return False
if not queue.link(depayloader):
log.error(f"Failed to link queue to depayloader for {media_type}")
self._cleanup_elements([queue, depayloader], self._recording_elements)
return False
# Link depayloader to muxer
depayloader_src_pad = depayloader.get_static_pad("src")
muxer_pad_name = "video_%u" if media_type == "video" else "audio_%u"
muxer_sink_pad = self._recording_muxer.get_request_pad(muxer_pad_name)
if not muxer_sink_pad:
log.error(f"Failed to get request pad '{muxer_pad_name}' from muxer")
self._cleanup_elements([queue, depayloader], self._recording_elements)
return False
link_result = depayloader_src_pad.link(muxer_sink_pad)
if link_result != Gst.PadLinkReturn.OK:
if hasattr(link_result, "value_nick"):
reason = link_result.value_nick
else:
reason = link_result
log.error(
f"Failed to link depayloader to muxer for {media_type} (reason: {reason})"
)
self._cleanup_elements([queue, depayloader], self._recording_elements)
self._recording_muxer.release_request_pad(muxer_sink_pad)
return False
queue.sync_state_with_parent()
depayloader.sync_state_with_parent()
self._recording_elements.extend([queue, depayloader])
log.debug(f"Successfully added {media_type} RTP recording branch")
return True
def _setup_stdin_source(self) -> None:
"""Setup stdin source with decodebin for automatic format detection."""
assert self.pipeline is not None
fdsrc = Gst.ElementFactory.make("fdsrc")
if not fdsrc:
log.error("Critical: Failed to create fdsrc element")
return
fdsrc.set_property("fd", 0)
# Make it a live source to run at real-time speed.
fdsrc.set_property("is-live", True)
fdsrc.set_property("do-timestamp", True)
# ``decodebin`` is used for automatic format detection.
decodebin = Gst.ElementFactory.make("decodebin")
if not decodebin:
log.error("Critical: Failed to create decodebin element")
return
self.pipeline.add(fdsrc)
self.pipeline.add(decodebin)
if not fdsrc.link(decodebin):
log.error("Failed to link fdsrc to decodebin")
return
decodebin.connect("pad-added", self._on_stdin_decodebin_stream)
fdsrc.sync_state_with_parent()
decodebin.sync_state_with_parent()
log.info("Successfully setup stdin source.")
def _on_stdin_decodebin_stream(self, decodebin, pad: Gst.Pad) -> None:
"""Handle streams from stdin decodebin."""
assert self.pipeline is not None
caps = pad.get_current_caps()
if not caps:
caps = pad.query_caps(None)
name = caps.get_structure(0).get_name()
log.info(f"Handling stdin decoded pad of type {name}")
log.info(f"Pad caps: {caps.to_string()}")
if name.startswith("video/"):
log.info("===> VIDEO (stdin)")
video_selector = self.pipeline.get_by_name("video_selector")
if not video_selector:
log.error("video_selector not found in pipeline")
return
log.debug("video_selector pads before:")
for p in video_selector.pads:
log.debug(f" {p.get_name()}: direction={p.direction}")
try:
active_pad = video_selector.get_property("active-pad")
current_active_pad = active_pad.get_name() if active_pad else "none"
log.debug(f"Current active pad: {current_active_pad}.")
except:
log.warning("Could not get active-pad property.")
# Synchronize buffers to pipeline clock to avoid video playing too fast.
clocksync = Gst.ElementFactory.make("clocksync")
if not clocksync:
log.warning("clocksync element not available, video may play too fast.")
else:
self.pipeline.add(clocksync)
log.debug("Created clocksync element for real-time playback")
queue = Gst.ElementFactory.make("queue")
if not queue:
log.error("Failed to create queue for stdin video")
return
queue.set_property("max-size-buffers", 30)
queue.set_property("max-size-bytes", 0)
queue.set_property("max-size-time", 2000000000)
self.pipeline.add(queue)
if clocksync:
sink_pad = clocksync.get_static_pad("sink")
link_result = pad.link(sink_pad)
if link_result != Gst.PadLinkReturn.OK:
log.error(
f"Failed to link decodebin pad to clocksync sink: {link_result}."
)
return
if not clocksync.link(queue):
log.error("Failed to link clocksync to queue")
return
else:
sink_pad = queue.get_static_pad("sink")
link_result = pad.link(sink_pad)
if link_result != Gst.PadLinkReturn.OK:
log.error(
f"Failed to link decodebin pad to queue sink: {link_result}."
)
return
src_pad = queue.get_static_pad("src")
sink_pad = video_selector.get_request_pad("sink_%u")
if not src_pad or not sink_pad:
log.error("Failed to get pads for linking queue to video_selector")
return
link_result = src_pad.link(sink_pad)
if link_result != Gst.PadLinkReturn.OK:
log.error(f"Failed to link queue to video_selector: {link_result}")
return
log.info("video_selector pads after linking:")
for p in video_selector.pads:
log.debug(f" {p.get_name()}: direction={p.direction}")
video_selector.set_property("active-pad", sink_pad)
log.debug("Set video_selector active pad")
try:
active_pad = video_selector.get_property("active-pad")
current_active_pad = active_pad.get_name() if active_pad else "none"
log.debug(f"New active pad: {current_active_pad}.")
except:
log.warning("Could not verify active-pad property")
queue.sync_state_with_parent()
state, _, _ = self.pipeline.get_state(0)
log.debug(f"Pipeline state: {state}.")
elif name.startswith("audio/"):
log.info("===> AUDIO (stdin)")
audio_valve = self.pipeline.get_by_name("audio_valve")
if not audio_valve:
log.error("audio_valve not found in pipeline")
return
# Synchronize buffers to pipeline clock to avoid audio playing too fast.
clocksync = Gst.ElementFactory.make("clocksync")
if not clocksync:
log.warning("clocksync element not available, audio may play too fast.")
else:
self.pipeline.add(clocksync)
log.debug("Created clocksync element for real-time audio playback")
queue = Gst.ElementFactory.make("queue")
if not queue:
log.error("Failed to create queue for stdin audio")
return
self.pipeline.add(queue)
if clocksync:
sink_pad = clocksync.get_static_pad("sink")
link_result = pad.link(sink_pad)
if link_result != Gst.PadLinkReturn.OK:
log.error(
f"Failed to link decodebin pad to clocksync sink: {link_result}"
)
return
if not clocksync.link(queue):
log.error("Failed to link clocksync to queue")
return
else:
sink_pad = queue.get_static_pad("sink")
link_result = pad.link(sink_pad)
if link_result != Gst.PadLinkReturn.OK:
log.error(
f"Failed to link decodebin pad to queue sink: {link_result}"
)
return
link_result = queue.link(audio_valve)
if not link_result:
log.error("Failed to link queue to audio_valve")
return
if clocksync:
clocksync.sync_state_with_parent()
queue.sync_state_with_parent()
log.debug("Audio elements synced to pipeline state")
state, _, _ = self.pipeline.get_state(0)
log.debug(f"Pipeline state: {state}.")
else:
log.warning(f"Ignoring unknown pad type from stdin decodebin: {name!r}.")
async def _call_start(self, callee: jid.JID, call_data: dict, profile: str) -> str:
return await self.bridge.call_start(
str(self.callee), data_format.serialise({"sdp": self.offer}), self.profile
)
async def _start_call(self) -> None:
"""Initiate the call.
Initiates a call with the callee using the stored offer. If there are any buffered
local ICE candidates, they are sent as part of the initiation.
"""
assert self.callee
assert self.call_start_cb is not None
self.sid = await self.call_start_cb(
self.callee, {"sdp": self.offer}, self.profile
)
if self.local_candidates_buffer:
log.debug(
f"sending buffered local ICE candidates: {self.local_candidates_buffer}"
)
if not self.pwd:
sdp = self.webrtcbin.props.local_description.sdp.as_text()
self.extract_ufrag_pwd(sdp)
ice_data = {}
for media_type, candidates in self.local_candidates_buffer.items():
if media_type not in self.ufrag or media_type not in self.pwd:
log.error(f"Missing ICE credentials for {media_type}")
log.debug(f"Available ufrag keys: {list(self.ufrag.keys())}")
log.debug(f"Available pwd keys: {list(self.pwd.keys())}")
continue
ice_data[media_type] = {
"ufrag": self.ufrag[media_type],
"pwd": self.pwd[media_type],
"candidates": candidates,
}
if ice_data:
log.debug(f"Sending ICE data for media types: {list(ice_data.keys())}")
await self.bridge.ice_candidates_add(
self.sid, data_format.serialise(ice_data), self.profile
)
else:
log.error("No ICE data to send - missing credentials!")
self.local_candidates_buffer.clear()
def _remote_sdp_set(self, promise) -> None:
"""Called after set-remote-description with an ANSWER is complete (initiator)."""
assert promise.wait() == Gst.PromiseResult.REPLIED
self.sdp_set = True
# The remote description is now set. We can process remote candidates and send our
# buffered ones.
self.remote_desc_set = True
def on_accepted_call(self, sdp: str, profile: str) -> None:
"""Outgoing call has been accepted.
@param sdp: The SDP answer string received from the other party.
@param profile: Profile used for the call.
"""
log.debug(f"SDP answer received: \n{sdp}")
__, sdpmsg = GstSdp.SDPMessage.new_from_text(sdp)
answer = GstWebRTC.WebRTCSessionDescription.new(
GstWebRTC.WebRTCSDPType.ANSWER, sdpmsg
)
promise = Gst.Promise.new_with_change_func(self._remote_sdp_set)
self.webrtcbin.emit("set-remote-description", answer, promise)
async def answer_call(self, sdp: str, profile: str) -> None:
"""Answer an incoming call
@param sdp: The SDP offer string received from the initiator.
@param profile: Profile used for the call.
@raise AssertionError: Raised when either "VP8" or "OPUS" is not present in
payload types.
"""
log.debug(f"SDP offer received: \n{sdp}")
self._set_media_types(sdp)
__, offer_sdp_msg = GstSdp.SDPMessage.new_from_text(sdp)
payload_types = self.get_payload_types(
offer_sdp_msg, video_encoding="VP8", audio_encoding="OPUS"
)
assert "VP8" in payload_types
assert "OPUS" in payload_types
try:
await self.setup_call(
"responder", audio_pt=payload_types["OPUS"], video_pt=payload_types["VP8"]
)
except Exception:
log.exception("Can't setup call")
raise
self.start_pipeline()
offer = GstWebRTC.WebRTCSessionDescription.new(
GstWebRTC.WebRTCSDPType.OFFER, offer_sdp_msg
)
promise = Gst.Promise.new_with_change_func(self.on_offer_set)
self.webrtcbin.emit("set-remote-description", offer, promise)
# For the responder, the remote description is set synchronously. We can set the
# flag here.
self.remote_desc_set = True
def _send_buffered_local_candidates(self) -> None:
"""Sends any local ICE candidates that were buffered before the remote description was set."""
if not self.local_candidates_buffer:
return
log.debug(
f"sending buffered local ICE candidates: {self.local_candidates_buffer}"
)
sdp = self.webrtcbin.props.local_description.sdp.as_text()
self.extract_ufrag_pwd(sdp)
ice_data = {}
for media_type, candidates in self.local_candidates_buffer.items():
if media_type not in self.ufrag or media_type not in self.pwd:
log.error(f"Missing ICE credentials for {media_type}")
continue
ice_data[media_type] = {
"ufrag": self.ufrag[media_type],
"pwd": self.pwd[media_type],
"candidates": candidates,
}
if ice_data:
log.debug(f"Sending ICE data for media types: {list(ice_data.keys())}")
self._a_call(
self.bridge.ice_candidates_add,
self.sid,
data_format.serialise(ice_data),
self.profile,
)
else:
log.error("No ICE data to send - missing credentials!")
self.local_candidates_buffer.clear()
def on_ice_candidate(self, webrtc, mline_index, candidate_sdp):
"""Handles the on-ice-candidate signal of webrtcbin.
@param webrtc: The webrtcbin element.
@param mlineindex: The mline index.
@param candidate: The ICE candidate.
"""
# Empty candidate signals end of candidates for this mline
if not candidate_sdp or candidate_sdp.strip() == "":
log.debug(f"ICE gathering complete for mline {mline_index}")
return
log.debug(
f"Local ICE candidate. MLine Index: {mline_index}, Candidate: {candidate_sdp}"
)
parsed_candidate = self.parse_ice_candidate(candidate_sdp)
if parsed_candidate is None:
log.warning(f"Can't parse candidate: {candidate_sdp}")
return
try:
media_type = self.media_types[mline_index]
except KeyError:
log.error(f"Can't find media type for mline {mline_index}")
log.debug(f"{self.media_types=}")
return
# Buffer local candidates until the remote description is set to avoid a race
# condition.
if not self._remote_desc_set:
log.debug(f"buffering local ICE candidate for {media_type}")
self.local_candidates_buffer.setdefault(media_type, []).append(
parsed_candidate
)
else:
# The remote description is set, so we can send the candidate immediately.
sdp = self.webrtcbin.props.local_description.sdp.as_text()
self.extract_ufrag_pwd(sdp)
ice_data = {
"ufrag": self.ufrag[media_type],
"pwd": self.pwd[media_type],
"candidates": [parsed_candidate],
}
self._a_call(
self.bridge.ice_candidates_add,
self.sid,
data_format.serialise({media_type: ice_data}),
self.profile,
)
def on_ice_candidates_new(self, candidates: dict) -> None:
"""Handle new ICE candidates.
@param candidates: A dictionary containing media types ("audio" or "video") as
keys and corresponding ICE data as values.
@raise exceptions.InternalError: Raised when sdp mline index is not found.
"""
if not self.sdp_set:
log.debug("buffering remote ICE candidate")
for media_type in ("audio", "video"):
media_candidates = candidates.get(media_type)
if media_candidates:
buffer = self.remote_candidates_buffer[media_type]
buffer["candidates"].extend(media_candidates["candidates"])
return
for media_type, ice_data in candidates.items():
for candidate in ice_data["candidates"]:
candidate_sdp = self.build_ice_candidate(candidate)
try:
mline_index = self.get_sdp_mline_index(media_type)
except Exception as e:
raise exceptions.InternalError(f"Can't find sdp mline index: {e}")
self.webrtcbin.emit("add-ice-candidate", mline_index, candidate_sdp)
log.debug(
f"Remote ICE candidate added. MLine Index: {mline_index}, "
f"Candidate: {candidate_sdp}"
)
def on_ice_gathering_state_change(self, pspec, __):
state = self.webrtcbin.get_property("ice-gathering-state")
log.debug(f"ICE gathering state changed to {state}")
def on_ice_connection_state(self, pspec, __):
state = self.webrtcbin.props.ice_connection_state
log.debug(f"ICE connection state changed to: {state}")
if state == GstWebRTC.WebRTCICEConnectionState.FAILED:
log.error("ICE connection failed - check firewall/STUN/TURN configuration")
elif state == GstWebRTC.WebRTCICEConnectionState.CONNECTED:
log.info("ICE connection established successfully")
elif state == GstWebRTC.WebRTCICEConnectionState.COMPLETED:
log.info("ICE connectivity checks completed")
def on_bus_error(self, bus: Gst.Bus, message: Gst.Message) -> None:
"""Handles the GStreamer bus error messages.
@param bus: The GStreamer bus.
@param message: The error message.
"""
err, debug = message.parse_error()
log.error(f"Error from {message.src.get_name()}: {err.message}")
log.error(f"Debugging info: {debug}")
def on_bus_eos(self, bus: Gst.Bus, message: Gst.Message) -> None:
"""Handles the GStreamer bus eos messages.
@param bus: The GStreamer bus.
@param message: The eos message.
"""
log.info("End of stream")
def on_audio_mute(self, muted: bool) -> None:
"""Handles (un)muting of audio.
@param muted: True if audio is muted.
"""
if self.audio_valve is None:
log.warning("No audio valve, can't mute.)")
else:
self.audio_valve.set_property("drop", muted)
state = "muted" if muted else "unmuted"
log.info(f"audio is now {state}")
def on_video_mute(self, muted: bool) -> None:
"""Handles (un)muting of video.
@param muted: True if video is muted.
"""
if self.video_selector is not None:
current_source = (
None if muted else "desktop" if self.desktop_sharing else "video"
)
self.switch_video_source(current_source)
state = "muted" if muted else "unmuted"
log.info(f"Video is now {state}")
def on_desktop_switch(self, desktop_active: bool) -> None:
"""Switches the video source between desktop and video.
@param desktop_active: True if desktop must be active. False for video.
"""
if desktop_active and self.desktop_portal is not None:
aio.run_async(self.on_desktop_switch_portal(desktop_active))
else:
self.do_desktop_switch(desktop_active)
async def on_desktop_switch_portal(self, desktop_active: bool) -> None:
"""Call freedesktop screenshare portal and the activate the shared stream"""
assert self.desktop_portal is not None
try:
screenshare_data = await self.desktop_portal.request_screenshare()
except exceptions.CancelError:
self.desktop_sharing = False
return
self.desktop_sharing_data = {"path": str(screenshare_data["node_id"])}
self.do_desktop_switch(desktop_active)
def on_portal_session_closed(self) -> None:
self.desktop_sharing = False
def do_desktop_switch(self, desktop_active: bool) -> None:
if self.video_muted:
# Update the active source state but do not switch
self.desktop_sharing = desktop_active
return
source = "desktop" if desktop_active else "video"
self.switch_video_source(source)
self.desktop_sharing = desktop_active
def switch_video_source(self, source: str | None) -> None:
"""Activates the specified source while deactivating the others.
@param source: 'desktop', 'video', 'muted' or None for muted source.
"""
assert self.pipeline is not None
if source is None:
source = "muted"
if source not in VIDEO_CONFIG:
raise ValueError(
f"Invalid source: {source!r}, use one of {VIDEO_CONFIG.keys()}"
)
self.set_pipeline_state(Gst.State.PAUSED)
vp8enc = self.pipeline.get_by_name("vp8enc")
if vp8enc is None:
raise exceptions.InternalError("Can't find vp8enc, it should be here.")
rate_caps_filter = self.pipeline.get_by_name("main_rate_caps")
if rate_caps_filter is None:
raise exceptions.InternalError(
"Can't find main_rate_caps, it should be here."
)
config = VIDEO_CONFIG[source]
vp8enc.set_property("target-bitrate", config.bitrate)
vp8enc.set_property("keyframe-max-dist", config.keyframe_max)
# Update the framerate for the entire encoding pipeline
new_caps = Gst.Caps.from_string(f"video/x-raw,framerate={config.fps}/1")
rate_caps_filter.set_property("caps", new_caps)
log.debug(f"Updated pipeline framerate to {config.fps} FPS")
# Create a new desktop source if necessary
if source == "desktop":
self._setup_desktop_source(self.desktop_sharing_data)
# Activate the chosen source and deactivate the others
for src_name in ["video", "muted", "desktop"]:
src_element = self.pipeline.get_by_name(f"{src_name}_src")
if src_name == source:
if src_element:
src_element.set_state(Gst.State.PLAYING)
else:
if src_element:
if src_name == "desktop":
self._remove_desktop_source(src_element)
else:
src_element.set_state(Gst.State.NULL)
# Set the video_selector active pad
if source == "desktop":
if self.desktop_sink_pad:
pad = self.desktop_sink_pad
else:
log.error(f"No desktop pad available")
pad = None
else:
pad_name = f"sink_{['video', 'muted'].index(source)}"
pad = self.video_selector.get_static_pad(pad_name)
if pad is not None:
self.video_selector.props.active_pad = pad
self.set_pipeline_state(Gst.State.PLAYING)
def _setup_desktop_source(self, properties: dict[str, object] | None) -> None:
"""Set up a new desktop source.
@param properties: The properties to set on the desktop source.
"""
log.debug("Setting up new desktop source...")
source_elt = "ximagesrc" if self.desktop_portal is None else "pipewiresrc"
desktop_src = Gst.ElementFactory.make(source_elt, "desktop_src")
if properties is None:
properties = {}
for key, value in properties.items():
log.debug(f"setting {source_elt} property: {key!r}={value!r}")
desktop_src.set_property(key, value)
video_scale = Gst.ElementFactory.make("videoscale", "desktop_videoscale")
video_convert = Gst.ElementFactory.make("videoconvert", "desktop_videoconvert")
video_rate = Gst.ElementFactory.make("videorate", "desktop_videorate")
queue = Gst.ElementFactory.make("queue", "desktop_queue")
queue.set_property("leaky", "downstream")
queue.set_property("max-size-buffers", 3)
queue.set_property("flush-on-eos", True)
desktop_elements = [desktop_src, video_scale, video_convert, video_rate, queue]
for element in desktop_elements:
self.pipeline.add(element)
# Link the elements in the following order:
# src → scale → convert → rate → queue
# This order ensures we scale the high-resolution buffer first, saving CPU,
# then convert it to a more standard format that videorate can handle easily.
if not desktop_src.link(video_scale):
log.error("Failed to link desktop_src to videoscale")
self._cleanup_elements(desktop_elements)
return
if not video_scale.link(video_convert):
log.error("Failed to link videoscale to videoconvert")
self._cleanup_elements(desktop_elements)
return
if not video_convert.link(video_rate):
log.error("Failed to link videoconvert to videorate")
self._cleanup_elements(desktop_elements)
return
if not video_rate.link(queue):
log.error("Failed to link videorate to queue")
self._cleanup_elements(desktop_elements)
return
sink_pad_template = self.video_selector.get_pad_template("sink_%u")
self.desktop_sink_pad = self.video_selector.request_pad(
sink_pad_template, None, None
)
if not self.desktop_sink_pad:
log.error("Failed to request a new sink pad from video_selector.")
self._cleanup_elements(desktop_elements)
return
log.debug(
f"Requested and received pad {self.desktop_sink_pad.get_name()} from video_selector."
)
queue_src_pad = queue.get_static_pad("src")
link_ret = queue_src_pad.link(self.desktop_sink_pad)
if link_ret != Gst.PadLinkReturn.OK:
log.error(f"Failed to link queue to video_selector: {link_ret}")
self._cleanup_elements(desktop_elements)
self.video_selector.release_request_pad(self.desktop_sink_pad)
self.desktop_sink_pad = None
return
# Sync states after successful linking
for element in desktop_elements:
element.sync_state_with_parent()
log.debug("Desktop source pipeline successfully created and linked.")
def _cleanup_elements(
self, elements: list[Gst.Element], elements_list: list[Gst.Element] | None = None
) -> None:
"""Helper to cleanup elements.
@param elements: List of elements to remove from pipeline
"""
if self.pipeline is None:
return
for element in elements:
try:
if element:
element.set_state(Gst.State.NULL)
self.pipeline.remove(element)
if elements_list and element in elements_list:
elements_list.remove(element)
except Exception as e:
log.warning(
f"Error cleaning up element {element.get_name() if element else 'Unknown'}: {e}"
)
def _remove_desktop_source(self, desktop_src: Gst.Element) -> None:
"""Remove the desktop source from the pipeline.
@param desktop_src: The desktop source to remove.
"""
# Remove elements for the desktop source
video_convert = self.pipeline.get_by_name("desktop_videoconvert")
video_rate = self.pipeline.get_by_name("desktop_videorate")
video_scale = self.pipeline.get_by_name("desktop_videoscale")
capsfilter = self.pipeline.get_by_name("desktop_capsfilter")
queue = self.pipeline.get_by_name("desktop_queue")
# Unlink and remove in reverse order
elements = [
queue,
capsfilter,
video_scale,
video_rate,
video_convert,
desktop_src,
]
self._cleanup_elements(elements)
# Release the pad associated with the desktop source
if self.desktop_sink_pad:
self.video_selector.release_request_pad(self.desktop_sink_pad)
self.desktop_sink_pad = None
if self.desktop_portal is not None:
self.desktop_portal.end_session()
async def end_call(self) -> None:
"""Stop streaming and clean instance"""
self.reset_instance()
class WebRTCCall:
"""Helper class to create and handle WebRTC.
This class handles signals and communication of connection data with backend.
"""
def __init__(
self,
bridge,
profile: str,
callee: jid.JID,
on_call_setup_cb: Callable | None = None,
on_call_ended_cb: Callable | None = None,
**kwargs,
):
"""Create and setup a webRTC instance
@param bridge: async Bridge.
@param profile: profile making or receiving the call
@param callee: peer jid
@param kwargs: extra kw args to use when instantiating WebRTC
"""
self.profile = profile
self.webrtc = WebRTC(bridge, profile, **kwargs)
self.webrtc.callee = callee
self.on_call_setup_cb = on_call_setup_cb
self.on_call_ended_cb = on_call_ended_cb
bridge.register_signal("ice_candidates_new", self.on_ice_candidates_new, "plugin")
bridge.register_signal("call_setup", self.on_call_setup, "plugin")
bridge.register_signal("call_ended", self.on_call_ended, "plugin")
@classmethod
async def make_webrtc_call(
cls, bridge, profile: str, call_data: CallData, **kwargs
) -> "WebRTCCall":
"""Create the webrtc_call instance
@param call_data: Call data of the command
@param kwargs: extra args used to instanciate WebRTCCall
"""
webrtc_call = cls(bridge, profile, call_data.callee, **call_data.kwargs, **kwargs)
if call_data.sid is None:
# we are making the call
await webrtc_call.start()
else:
# we are receiving the call
webrtc_call.sid = call_data.sid
if call_data.action_id is not None:
await bridge.action_launch(
call_data.action_id,
data_format.serialise({"cancelled": False}),
profile,
)
return webrtc_call
@property
def sid(self) -> str | None:
return self.webrtc.sid
@sid.setter
def sid(self, new_sid: str | None) -> None:
self.webrtc.sid = new_sid
async def on_ice_candidates_new(
self, sid: str, candidates_s: str, profile: str
) -> None:
if sid != self.webrtc.sid or profile != self.profile:
return
self.webrtc.on_ice_candidates_new(
data_format.deserialise(candidates_s),
)
async def on_call_setup(self, sid: str, setup_data_s: str, profile: str) -> None:
if sid != self.webrtc.sid or profile != self.profile:
return
setup_data = data_format.deserialise(setup_data_s)
try:
role = setup_data["role"]
sdp = setup_data["sdp"]
except KeyError:
log.error(f"Invalid setup data received: {setup_data}")
return
if role == "initiator":
self.webrtc.on_accepted_call(sdp, profile)
elif role == "responder":
await self.webrtc.answer_call(sdp, profile)
else:
log.error(f"Invalid role received during setup: {setup_data}")
if self.on_call_setup_cb is not None:
await aio.maybe_async(self.on_call_setup_cb(sid, profile))
async def on_call_ended(self, sid: str, data_s: str, profile: str) -> None:
if sid != self.webrtc.sid or profile != self.profile:
return
await self.webrtc.end_call()
if self.on_call_ended_cb is not None:
await aio.maybe_async(self.on_call_ended_cb(sid, profile))
async def start(self):
"""Start a call.
To be used only if we are initiator
"""
try:
await self.webrtc.setup_call("initiator")
except Exception:
log.exception("Can't setup call")
raise
self.webrtc.start_pipeline()