core: Replace legacy INI config system with Pydantic-based TOML configuration.

This is a huge configuration refactoring.

The configuration system now uses Pydantic settings, with better validation, better
support for environment variables and other goodies.

The main config files now use TOML instead of INI, this allow for better handling of
complex types, notably dict.

Plugins, components and frontends can declare their config settings in a class which will
be imported dynamically.
This commit is contained in:
Goffi 2025-11-30 12:27:35 +01:00
commit 6a7b7bc36a
4 changed files with 1392 additions and 1011 deletions

View file

@ -53,6 +53,7 @@ from twisted.words.protocols.jabber import error, jid
from twisted.words.xish import domish
from wokkel import pubsub, rsm
from libervia.backend import G
from libervia.backend.core import exceptions
from libervia.backend.core.constants import Const as C
from libervia.backend.core.core_types import SatXMPPEntity
@ -66,14 +67,15 @@ from libervia.backend.tools.common import data_format, tls, uri
from libervia.backend.tools.common.async_utils import async_lru
from .ad_hoc import APAdHocService
from .events import APEvents
# We import PLUGIN_INFO as it's needed for the plugin, and we put it in .constants because
# it's necessary to create the ComponentSettings subclass.
from .constants import PLUGIN_INFO
from .constants import (
ACTIVITY_OBJECT_MANDATORY,
ACTIVITY_TARGET_MANDATORY,
ACTIVITY_TYPES,
ACTIVITY_TYPES_LOWER,
COMMENTS_MAX_PARENTS,
CONF_SECTION,
IMPORT_NAME,
LRU_MAX_SIZE,
MEDIA_TYPE_AP,
@ -84,55 +86,23 @@ from .constants import (
TYPE_EVENT,
TYPE_FOLLOWERS,
TYPE_ITEM,
TYPE_JOIN,
TYPE_LEAVE,
TYPE_LIKE,
TYPE_MENTION,
TYPE_REACTION,
TYPE_TOMBSTONE,
TYPE_JOIN,
TYPE_LEAVE,
)
from .events import APEvents
from .http_server import HTTPServer
from .pubsub_service import APPubsubService
from .regex import RE_MENTION
from .settings import APGatewaySettings
log = getLogger(__name__)
IMPORT_NAME = "ap-gateway"
PLUGIN_INFO = {
C.PI_NAME: "ActivityPub Gateway component",
C.PI_IMPORT_NAME: IMPORT_NAME,
C.PI_MODES: [C.PLUG_MODE_COMPONENT],
C.PI_TYPE: C.PLUG_TYPE_ENTRY_POINT,
C.PI_PROTOCOLS: [],
C.PI_DEPENDENCIES: [
"XEP-0050",
"XEP-0054",
"XEP-0060",
"XEP-0084",
"XEP-0106",
"XEP-0277",
"XEP-0292",
"XEP-0329",
"XEP-0372",
"XEP-0424",
"XEP-0465",
"XEP-0470",
"XEP-0447",
"XEP-0471",
"PUBSUB_CACHE",
"TEXT_SYNTAXES",
"IDENTITY",
],
C.PI_RECOMMENDATIONS: [],
C.PI_MAIN: "APGateway",
C.PI_HANDLER: C.BOOL_TRUE,
C.PI_DESCRIPTION: _(
"Gateway for bidirectional communication between XMPP and ActivityPub."
),
}
HEXA_ENC = r"(?P<hex>[0-9a-fA-f]{2})"
RE_PERIOD_ENC = re.compile(f"\\.{HEXA_ENC}")
RE_PERCENT_ENC = re.compile(f"%{HEXA_ENC}")
@ -181,6 +151,7 @@ class APGateway:
self.pubsub_service = APPubsubService(self)
self.ad_hoc = APAdHocService(self)
self.ap_events = APEvents(self)
self.html_redirect: HTMLRedirectDict = {}
host.trigger.add_with_check(
"message_received", self, self._message_received_trigger, priority=-1000
)
@ -242,14 +213,19 @@ class APGateway:
# params
# URL and port
self.public_url = self.host.memory.config_get(
CONF_SECTION, "public_url"
) or self.host.memory.config_get(CONF_SECTION, "xmpp_domain")
# Get component settings
ap_settings = G.config.get_settings(APGatewaySettings)
self.public_url = (
ap_settings.public_url
or ap_settings.xmpp_domain
or G.config.common.xmpp_domain
)
if self.public_url is None:
log.error(
'"public_url" not set in configuration, this is mandatory to have'
"ActivityPub Gateway running. Please set this option it to public facing "
f"url in {CONF_SECTION!r} configuration section."
f"url in {IMPORT_NAME!r} configuration section."
)
return
if parse.urlparse(self.public_url).scheme:
@ -258,65 +234,34 @@ class APGateway:
'"public_url" configuration option. ActivityPub Gateway won\'t be run.'
)
return
self.http_port = int(self.host.memory.config_get(CONF_SECTION, "http_port", 8123))
connection_type = self.host.memory.config_get(
CONF_SECTION, "http_connection_type", "https"
)
self.http_port = ap_settings.http_port
connection_type = ap_settings.http_connection_type
if connection_type not in ("http", "https"):
raise exceptions.ConfigError(
'bad ap-gateay http_connection_type, you must use one of "http" or '
'"https"'
)
self.http_sign_get = C.bool(
self.host.memory.config_get(CONF_SECTION, "http_sign_get", C.BOOL_TRUE)
)
self.max_items = int(
self.host.memory.config_get(CONF_SECTION, "new_node_max_items", 50)
)
self.comments_max_depth = int(
self.host.memory.config_get(CONF_SECTION, "comments_max_depth", 0)
)
self.ap_path = self.host.memory.config_get(CONF_SECTION, "ap_path", "_ap")
self.http_sign_get = ap_settings.http_sign_get
self.max_items = ap_settings.new_node_max_items
self.comments_max_depth = ap_settings.comments_max_depth
self.ap_path = ap_settings.ap_path
self.base_ap_url = parse.urljoin(f"https://{self.public_url}", f"{self.ap_path}/")
# True (default) if we provide gateway only to entities/services from our server
self.local_only = C.bool(
self.host.memory.config_get(CONF_SECTION, "local_only", C.BOOL_TRUE)
)
self.local_only = ap_settings.local_only
# if True (default), mention will be parsed in non-private content coming from
# XMPP. This is necessary as XEP-0372 are coming separately from item where the
# mention is done, which is hard to impossible to translate to ActivityPub (where
# mention specified inside the item directly). See documentation for details.
self.auto_mentions = C.bool(
self.host.memory.config_get(CONF_SECTION, "auto_mentions", C.BOOL_TRUE)
)
self.auto_mentions = ap_settings.auto_mentions
html_redirect: Dict[str, Union[str, dict]] = self.host.memory.config_get(
CONF_SECTION, "html_redirect_dict", {}
)
self.html_redirect: Dict[str, List[dict]] = {}
for url_type, target in html_redirect.items():
if isinstance(target, str):
target = {"url": target}
elif not isinstance(target, dict):
raise exceptions.ConfigError(
f"html_redirect target must be a URL or a dict, not {target!r}"
)
filters = target.setdefault("filters", {})
if "url" not in target:
log.warning(f"invalid HTML redirection, missing target URL: {target}")
continue
# a slash in the url_type is a syntactic shortcut to have a node filter
if "/" in url_type:
url_type, node_filter = url_type.split("/", 1)
filters["node"] = node_filter
self.html_redirect.setdefault(url_type, []).append(target)
self.html_redirect = ap_settings.html_redirect
# HTTP server launch
self.server = HTTPServer(self)
if connection_type == "http":
reactor.listenTCP(self.http_port, self.server)
else:
options = tls.get_options_from_config(self.host.memory.config, CONF_SECTION)
options = tls.get_options_from_config(ap_settings.tls)
tls.tls_options_check(options)
context_factory = tls.get_tls_context_factory(options)
reactor.listenSSL(self.http_port, self.server, context_factory)
@ -1596,8 +1541,10 @@ class APGateway:
elif start_index > 5000:
raise error.StanzaError(
"feature-not-implemented",
text="Maximum limit for previous_index has been reached, this limit"
"is set to avoid DoS",
text=(
"Maximum limit for previous_index has been reached, this limit"
"is set to avoid DoS"
),
)
else:
# we'll convert "start_index" to "after_id", thus we need the item just
@ -1972,7 +1919,7 @@ class APGateway:
except StopIteration:
# no item corresponding to this author, we use ap_account
log.warning(
"Can't find a single cached item for parent item " f"{parent_item!r}"
f"Can't find a single cached item for parent item {parent_item!r}"
)
parent_ap_account = ap_account
else:
@ -2419,7 +2366,7 @@ class APGateway:
if parsed_anchor["type"] != "pubsub":
log.warning(
f"ignoring reference with non pubsub anchor, this is not supported: "
"{reference_data!r}"
f"{{reference_data!r}}"
)
return False

View file

@ -17,8 +17,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from libervia.backend.core.constants import Const as C
from libervia.backend.core.i18n import _
IMPORT_NAME = "ap-gateway"
CONF_SECTION = f"component {IMPORT_NAME}"
CONTENT_TYPE_WEBFINGER = "application/jrd+json; charset=utf-8"
CONTENT_TYPE_AP = "application/activity+json; charset=utf-8"
TYPE_ACTOR = "actor"
@ -129,3 +131,36 @@ MAX_AVATAR_SIZE = 1024 * 1024 * 5
# storage prefixes
ST_AVATAR = "[avatar]"
ST_AP_CACHE = "[AP_item_cache]"
PLUGIN_INFO = {
C.PI_NAME: "ActivityPub Gateway component",
C.PI_IMPORT_NAME: IMPORT_NAME,
C.PI_MODES: [C.PLUG_MODE_COMPONENT],
C.PI_TYPE: C.PLUG_TYPE_ENTRY_POINT,
C.PI_PROTOCOLS: [],
C.PI_DEPENDENCIES: [
"XEP-0050",
"XEP-0054",
"XEP-0060",
"XEP-0084",
"XEP-0106",
"XEP-0277",
"XEP-0292",
"XEP-0329",
"XEP-0372",
"XEP-0424",
"XEP-0465",
"XEP-0470",
"XEP-0447",
"XEP-0471",
"PUBSUB_CACHE",
"TEXT_SYNTAXES",
"IDENTITY",
],
C.PI_RECOMMENDATIONS: [],
C.PI_MAIN: "APGateway",
C.PI_HANDLER: C.BOOL_TRUE,
C.PI_DESCRIPTION: _(
"Gateway for bidirectional communication between XMPP and ActivityPub."
),
}

View file

@ -18,7 +18,7 @@
import time
import html
from typing import Optional, Dict, List, Any
from typing import TYPE_CHECKING, Optional, Dict, List, Any
import json
from urllib import parse
from collections import deque
@ -40,6 +40,9 @@ from libervia.backend.core.log import getLogger
from libervia.backend.tools.common import date_utils, uri
from libervia.backend.memory.sqla_mapping import SubscriptionState
if TYPE_CHECKING:
from . import APGateway
from .constants import (
NS_AP,
MEDIA_TYPE_AP,
@ -80,7 +83,7 @@ class HTTPAPGServer(web_resource.Resource):
isLeaf = True
def __init__(self, ap_gateway):
def __init__(self, ap_gateway: "APGateway"):
self.apg = ap_gateway
self._seen_digest = deque(maxlen=50)
super().__init__()
@ -1064,7 +1067,7 @@ class HTTPAPGServer(web_resource.Resource):
redirections = self.apg.html_redirect[request_type]
for redirection in redirections:
filters = redirection["filters"]
filters = redirection.filters
if not filters:
break
# if we have filter, they must all match
@ -1076,7 +1079,7 @@ class HTTPAPGServer(web_resource.Resource):
if redirection is not None:
kw = {k: parse.quote(str(v), safe="") for k, v in kw.items()}
target_url = redirection["url"].format(**kw)
target_url = redirection.url.format(**kw)
content = web_util.redirectTo(target_url.encode(), request)
request.write(content)
request.finish()

View file

@ -0,0 +1,226 @@
#!/usr/bin/env python3
# Libervia ActivityPub Gateway
# Copyright (C) 2009-2025 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from typing import Any, Literal
from pydantic import BaseModel, Field, field_validator
from libervia.backend.tools.config import ComponentSettings, TLSSettings
from .constants import PLUGIN_INFO
HTMLRedirectDict = dict[
Literal["actor", "item", "followers", "following"], list["HTMLRedictData"]
]
class HTMLRedictData(BaseModel):
url: str = Field(description="Target URL for the redirect")
filters: dict[Literal["jid", "jid_user", "node", "item"], str] = Field(
default_factory=dict,
description=("Filters to apply when matching redirection rules."),
)
class APGatewaySettings(ComponentSettings, plugin_info=PLUGIN_INFO):
# HTTP server configuration
http_port: int = Field(
default=8123,
description="""
port where the HTTP server should listen. Port ``80`` is not used directly as it would
require root privileges, and it is strongly recommended against launching Libervia
under a privileged account. An HTTP Proxy or a port redirection should be set to
redirect the ``80`` port to the port specified here.
""",
)
http_connection_type: Literal["http", "https"] = Field(
default="https",
description="""
either ``http`` or ``https``. If you have a HTTP proxy such as Apache or NGINX which
already handles HTTPS, you may want to use ``http``.
Note that the HTTP server should always use ``https`` with end-user, the ``http``
option is only to be used with an HTTPS proxy.
""",
)
http_sign_get: bool = Field(
default=True,
description="""
A boolean value indicating if HTTP ``GET`` request made to AP implementations must be
signed. Some AP implementations require those requests to be signed, notably Mastodon
when "secure mode" is activated.
""",
)
# TLS
tls: TLSSettings | None = None
# ActivityPub configuration
public_url: str | None = Field(
default=None,
description=""""
Main user-facing domain of the HTTP server, this will be used to construct all AP URLs
**default**: if not set, ``xmpp_domain`` is used. If ``xmpp_domain`` is not set
either, an error is raised.
""",
)
xmpp_domain: str | None = Field(
default=None, description="XMPP domain. Default to ``common.xmpp_domain``."
)
ap_path: str = Field(
default="_ap",
description="""
Path prefix to use for ActivityPub request. It's usually not necessary to change the
default value.
""",
)
local_only: bool = Field(
default=True,
description="""
A boolean value indicating if the gateway is allowed to convert pubsub node from
external XMPP service or not. A JID is considered external if its domain part doesn't
end with the gateway's server. For instance, if a gateway ``ap.example.org`` is set on
the server ``example.org``, the JIDs ``pierre@example.org`` or
``some-node@pubsub.example.org`` will be considered local, but
``buenaventura@example.net`` won't (note the different domain).
Most of time, ``local_only`` should be used.
""",
)
auto_mentions: bool = Field(
default=True,
description="""
A boolean value indicating if received XMPP pubsub blog items bodies must be scanned
to find ``@user@server.tld`` type mentions. If mentions are found, they'll be added to
the resulting AP items.
""",
)
# PubSub configuration
new_node_max_items: int = Field(
default=50, description="Maximum items for new pubsub nodes"
)
comments_max_depth: int = Field(
default=0,
description="""
An integer value indicating the maximum number of comment nodes that can be created.
See :ref:`ap-xmpp-threads-conversion`
""",
)
# HTML redirection configuration
html_redirect: HTMLRedirectDict = Field(
default_factory=HTMLRedirectDict,
description="""
A dictionary used to redirect HTTP requests when it's not an ActivityPub request (i.e.
when the ``Accept`` header is not set to ``application/json``). Several ActivityPub
implementations link to original server when clicking on some links such as post
author or item ID, and this result in a request to the corresponding ActivityPub
endpoint, but made to retrieve HTML instead of ActivityPub JSON data.
By default, this request is showing the corresponding AP JSON data, but if you set
HTML redirection, you can redirect the page to your XMPP client web frontend (or
anything making sense), which results in better user experience.
The expected dictionary is a mapping URL request types to destination URL.
The URL request types is the first path element after ``ap_path``, it can be:
``actor``
Actor information. It should link to the profile page of the corresponding XMPP
entity
``item``
A specific publication. It should link to the rendered item
``followers``
Entities publicly subscribed to the actor. It is usually not used for redirection.
``following``
Entities that the actor is publicly subscribed to. It is usually not used for
redirection.
In the target URL you can uses template values in curly braces (something like
``{item}``). The values that you can use are:
``jid``
Full JID corresponding to the AP actor.
``jid_user``
Only the user part of the JID (what is before the ``@``)
``node``
Name of the pubsub node corresponding to the AP actor.
``item``
ID of the XMPP pubsub item.
You can use a slash ``/`` followed by a part of a node name to filter only on specific
nodes. For instance, if you want to redirect microblog items to a different URL that
event items, you can use ``item/urn:xmpp:microblog:0`` and ``item/urn:xmpp:events:0``,
the first redirection to match will be used. Note that anything containing the filter
value will match, thus you can also use ``item/microblog`` (but it then would match an
event which has ``microblog`` in is node name).
You can also use a dictionary as value instead of the target URL. This can be useful
to filter on an other value than the node name, if you want to make specific
redirection for a particular user for instance. If you use a dictionary, you must use
a ``url`` key with the target URL (which may have template values as usual), and you
may use a ``filters`` key mapping to an other dictionary where each key is a filter on
a specific template value (in other words: key is the template value, like ``jid`` or
``node``, and value is the filter to match).
*examples*
Redirect actor page to ``/profile/<JID user part>``, blogs items to ``/blog/<full
JID>/<node name>/<item id>`` and events items to ``/event/<full JID>/<node name>/<item
id>`` ::
[components.ap_gateway]
html_redirect.actor = "/profile/{jid_user}"
html_redirect.item = [
{node="microblog", url="/blog/{jid}/{node}/{item}"},
{node="events", url="/event/{jid}/{node}/{item}"},
]
Redirect items of user ``louise@example.org`` to ``/b/<item>``::
[components.ap_gateway]
html_redirect.item = {url = "/b/{item}", filters = { jid = "louise@example.org" }}
""",
)
@field_validator(
"html_redirect",
mode="before",
json_schema_input_type=str | HTMLRedictData | list[HTMLRedictData],
)
@classmethod
def html_redirect_parse_simple_syntax(cls, html_redirect: Any) -> Any:
if isinstance(html_redirect, dict):
for key, value in html_redirect.items():
if isinstance(value, str):
html_redirect[key] = [HTMLRedictData(url=value)]
elif isinstance(value, HTMLRedictData):
html_redirect[key] = [value]
elif isinstance(value, dict):
html_redirect[key] = [HTMLRedictData(**value)]
return html_redirect