refactoring: renamed sat package to libervia.backend

libervia/backend/plugins/plugin_comp_ap_gateway/ad_hoc.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+# Libervia ActivityPub Gateway
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from twisted.words.protocols.jabber import jid
+from twisted.words.xish import domish
+from wokkel import data_form
+
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.core_types import SatXMPPEntity
+from libervia.backend.core.i18n import _
+from libervia.backend.core.log import getLogger
+
+
+log = getLogger(__name__)
+NS_XMPP_JID_NODE_2_AP = "https://libervia.org/ap_gateway/xmpp_jid_node_2_ap_actor"
+
+class APAdHocService:
+    """Ad-Hoc commands for AP Gateway"""
+
+    def __init__(self, apg):
+        self.host = apg.host
+        self.apg = apg
+        self._c = self.host.plugins["XEP-0050"]
+
+    def init(self, client: SatXMPPEntity) -> None:
+        self._c.add_ad_hoc_command(
+            client,
+            self.xmpp_jid_node_2_ap_actor,
+            "Convert XMPP JID/Node to AP actor",
+            node=NS_XMPP_JID_NODE_2_AP,
+            allowed_magics=C.ENTITY_ALL,
+        )
+
+    async def xmpp_jid_node_2_ap_actor(
+        self,
+        client: SatXMPPEntity,
+        command_elt: domish.Element,
+        session_data: dict,
+        action: str,
+        node: str
+    ):
+        try:
+            x_elt = next(command_elt.elements(data_form.NS_X_DATA, "x"))
+            command_form = data_form.Form.fromElement(x_elt)
+        except StopIteration:
+            command_form = None
+        if command_form is None or len(command_form.fields) == 0:
+            # root request
+            status = self._c.STATUS.EXECUTING
+            form = data_form.Form(
+                "form", title="XMPP JID/node to AP actor conversion",
+                formNamespace=NS_XMPP_JID_NODE_2_AP
+            )
+
+            field = data_form.Field(
+                "text-single", "jid", required=True
+            )
+            form.addField(field)
+
+            field = data_form.Field(
+                "text-single", "node", required=False
+            )
+            form.addField(field)
+
+            payload = form.toElement()
+            return payload, status, None, None
+        else:
+            xmpp_jid = jid.JID(command_form["jid"])
+            xmpp_node = command_form.get("node")
+            actor = await self.apg.get_ap_account_from_jid_and_node(xmpp_jid, xmpp_node)
+            note = (self._c.NOTE.INFO, actor)
+            status = self._c.STATUS.COMPLETED
+            payload = None
+            return (payload, status, None, note)

libervia/backend/plugins/plugin_comp_ap_gateway/constants.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+# Libervia ActivityPub Gateway
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+IMPORT_NAME = "ap-gateway"
+CONF_SECTION = f"component {IMPORT_NAME}"
+CONTENT_TYPE_AP = "application/activity+json; charset=utf-8"
+TYPE_ACTOR = "actor"
+TYPE_INBOX = "inbox"
+TYPE_SHARED_INBOX = "shared_inbox"
+TYPE_OUTBOX = "outbox"
+TYPE_FOLLOWERS = "followers"
+TYPE_FOLLOWING = "following"
+TYPE_ITEM = "item"
+TYPE_TOMBSTONE = "Tombstone"
+TYPE_MENTION = "Mention"
+TYPE_LIKE = "Like"
+TYPE_REACTION = "EmojiReact"
+TYPE_EVENT = "Event"
+TYPE_JOIN = "Join"
+TYPE_LEAVE = "Leave"
+MEDIA_TYPE_AP = "application/activity+json"
+NS_AP = "https://www.w3.org/ns/activitystreams"
+NS_AP_PUBLIC = f"{NS_AP}#Public"
+# 3 values can be used, see https://www.w3.org/TR/activitypub/#public-addressing
+PUBLIC_TUPLE = (NS_AP_PUBLIC, "as:Public", "Public")
+AP_REQUEST_TYPES = {
+    "GET": {TYPE_ACTOR, TYPE_OUTBOX, TYPE_FOLLOWERS, TYPE_FOLLOWING},
+    "POST": {"inbox"},
+}
+AP_REQUEST_TYPES["HEAD"] = AP_REQUEST_TYPES["GET"]
+# headers to check for signature
+SIGN_HEADERS = {
+    # headers needed for all HTTP methods
+    None: [
+        # tuples are equivalent headers/pseudo headers, one of them must be present
+        ("date", "(created)"),
+        ("digest", "(request-target)"),
+    ],
+    b"GET": ["host"],
+    b"POST": ["digest"]
+}
+PAGE_SIZE = 10
+HS2019 = "hs2019"
+# delay after which a signed request is not accepted anymore
+SIGN_EXP = 12*60*60  # 12 hours (same value as for Mastodon)
+
+LRU_MAX_SIZE = 200
+ACTIVITY_TYPES = (
+    "Accept", "Add", "Announce", "Arrive", "Block", "Create", "Delete", "Dislike", "Flag",
+    "Follow", "Ignore", "Invite", "Join", "Leave", "Like", "Listen", "Move", "Offer",
+    "Question", "Reject", "Read", "Remove", "TentativeReject", "TentativeAccept",
+    "Travel", "Undo", "Update", "View",
+    # non-standard activities
+    "EmojiReact"
+)
+ACTIVITY_TYPES_LOWER = [a.lower() for a in ACTIVITY_TYPES]
+ACTIVITY_OBJECT_MANDATORY = (
+    "Create", "Update", "Delete", "Follow", "Add", "Remove", "Like", "Block", "Undo"
+)
+ACTIVITY_TARGET_MANDATORY = ("Add", "Remove")
+# activities which can be used with Shared Inbox (i.e. with no account specified)
+# must be lowercase
+ACTIVIY_NO_ACCOUNT_ALLOWED = (
+    "create", "update", "delete", "announce", "undo", "like", "emojireact", "join",
+    "leave"
+)
+# maximum number of parents to retrieve when comments_max_depth option is set
+COMMENTS_MAX_PARENTS = 100
+# maximum size of avatar, in bytes
+MAX_AVATAR_SIZE = 1024 * 1024 * 5
+
+# storage prefixes
+ST_AVATAR = "[avatar]"
+ST_AP_CACHE = "[AP_item_cache]"

libervia/backend/plugins/plugin_comp_ap_gateway/events.py

@@ -0,0 +1,407 @@
+#!/usr/bin/env python3
+
+# Libervia ActivityPub Gateway
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import Tuple
+
+import mimetypes
+import html
+
+import shortuuid
+from twisted.words.xish import domish
+from twisted.words.protocols.jabber import jid
+
+from libervia.backend.core.i18n import _
+from libervia.backend.core.log import getLogger
+from libervia.backend.core import exceptions
+from libervia.backend.tools.common import date_utils, uri
+
+from .constants import NS_AP_PUBLIC, TYPE_ACTOR, TYPE_EVENT, TYPE_ITEM
+
+
+log = getLogger(__name__)
+
+# direct copy of what Mobilizon uses
+AP_EVENTS_CONTEXT = {
+    "@language": "und",
+    "Hashtag": "as:Hashtag",
+    "PostalAddress": "sc:PostalAddress",
+    "PropertyValue": "sc:PropertyValue",
+    "address": {"@id": "sc:address", "@type": "sc:PostalAddress"},
+    "addressCountry": "sc:addressCountry",
+    "addressLocality": "sc:addressLocality",
+    "addressRegion": "sc:addressRegion",
+    "anonymousParticipationEnabled": {"@id": "mz:anonymousParticipationEnabled",
+                                      "@type": "sc:Boolean"},
+    "category": "sc:category",
+    "commentsEnabled": {"@id": "pt:commentsEnabled",
+                        "@type": "sc:Boolean"},
+    "discoverable": "toot:discoverable",
+    "discussions": {"@id": "mz:discussions", "@type": "@id"},
+    "events": {"@id": "mz:events", "@type": "@id"},
+    "ical": "http://www.w3.org/2002/12/cal/ical#",
+    "inLanguage": "sc:inLanguage",
+    "isOnline": {"@id": "mz:isOnline", "@type": "sc:Boolean"},
+    "joinMode": {"@id": "mz:joinMode", "@type": "mz:joinModeType"},
+    "joinModeType": {"@id": "mz:joinModeType",
+                     "@type": "rdfs:Class"},
+    "location": {"@id": "sc:location", "@type": "sc:Place"},
+    "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+    "maximumAttendeeCapacity": "sc:maximumAttendeeCapacity",
+    "memberCount": {"@id": "mz:memberCount", "@type": "sc:Integer"},
+    "members": {"@id": "mz:members", "@type": "@id"},
+    "mz": "https://joinmobilizon.org/ns#",
+    "openness": {"@id": "mz:openness", "@type": "@id"},
+    "participantCount": {"@id": "mz:participantCount",
+                         "@type": "sc:Integer"},
+    "participationMessage": {"@id": "mz:participationMessage",
+                             "@type": "sc:Text"},
+    "postalCode": "sc:postalCode",
+    "posts": {"@id": "mz:posts", "@type": "@id"},
+    "propertyID": "sc:propertyID",
+    "pt": "https://joinpeertube.org/ns#",
+    "remainingAttendeeCapacity": "sc:remainingAttendeeCapacity",
+    "repliesModerationOption": {"@id": "mz:repliesModerationOption",
+                                "@type": "mz:repliesModerationOptionType"},
+    "repliesModerationOptionType": {"@id": "mz:repliesModerationOptionType",
+                                    "@type": "rdfs:Class"},
+    "resources": {"@id": "mz:resources", "@type": "@id"},
+    "sc": "http://schema.org#",
+    "streetAddress": "sc:streetAddress",
+    "timezone": {"@id": "mz:timezone", "@type": "sc:Text"},
+    "todos": {"@id": "mz:todos", "@type": "@id"},
+    "toot": "http://joinmastodon.org/ns#",
+    "uuid": "sc:identifier",
+    "value": "sc:value"
+}
+
+
+class APEvents:
+    """XMPP Events <=> AP Events conversion"""
+
+    def __init__(self, apg):
+        self.host = apg.host
+        self.apg = apg
+        self._events = self.host.plugins["XEP-0471"]
+
+    async def event_data_2_ap_item(
+        self, event_data: dict, author_jid: jid.JID, is_new: bool=True
+    ) -> dict:
+        """Convert event data to AP activity
+
+        @param event_data: event data as used in [plugin_exp_events]
+        @param author_jid: jid of the published of the event
+        @param is_new: if True, the item is a new one (no instance has been found in
+            cache).
+            If True, a "Create" activity will be generated, otherwise an "Update" one will
+            be
+        @return: AP activity wrapping an Event object
+        """
+        if not event_data.get("id"):
+            event_data["id"] = shortuuid.uuid()
+        ap_account = await self.apg.get_ap_account_from_jid_and_node(
+            author_jid,
+            self._events.namespace
+        )
+        url_actor = self.apg.build_apurl(TYPE_ACTOR, ap_account)
+        url_item = self.apg.build_apurl(TYPE_ITEM, ap_account, event_data["id"])
+        ap_object = {
+            "actor": url_actor,
+            "attributedTo": url_actor,
+            "to": [NS_AP_PUBLIC],
+            "id": url_item,
+            "type": TYPE_EVENT,
+            "name": next(iter(event_data["name"].values())),
+            "startTime": date_utils.date_fmt(event_data["start"], "iso"),
+            "endTime": date_utils.date_fmt(event_data["end"], "iso"),
+            "url": url_item,
+        }
+
+        attachment = ap_object["attachment"] = []
+
+        # FIXME: we only handle URL head-picture for now
+        # TODO: handle jingle and use file metadata
+        try:
+            head_picture_url = event_data["head-picture"]["sources"][0]["url"]
+        except (KeyError, IndexError, TypeError):
+            pass
+        else:
+            media_type = mimetypes.guess_type(head_picture_url, False)[0] or "image/jpeg"
+            attachment.append({
+                "name": "Banner",
+                "type": "Document",
+                "mediaType": media_type,
+                "url": head_picture_url,
+            })
+
+        descriptions = event_data.get("descriptions")
+        if descriptions:
+            for description in descriptions:
+                content = description["description"]
+                if description["type"] == "xhtml":
+                    break
+            else:
+                content = f"<p>{html.escape(content)}</p>"  # type: ignore
+            ap_object["content"] = content
+
+        categories = event_data.get("categories")
+        if categories:
+            tag = ap_object["tag"] = []
+            for category in categories:
+                tag.append({
+                    "name": f"#{category['term']}",
+                    "type": "Hashtag",
+                })
+
+        locations = event_data.get("locations")
+        if locations:
+            ap_loc = ap_object["location"] = {}
+            # we only use the first found location
+            location = locations[0]
+            for source, dest in (
+                ("description", "name"),
+                ("lat", "latitude"),
+                ("lon", "longitude"),
+            ):
+                value = location.get(source)
+                if value is not None:
+                    ap_loc[dest] = value
+            for source, dest in (
+                ("country", "addressCountry"),
+                ("locality", "addressLocality"),
+                ("region", "addressRegion"),
+                ("postalcode", "postalCode"),
+                ("street", "streetAddress"),
+            ):
+                value = location.get(source)
+                if value is not None:
+                    ap_loc.setdefault("address", {})[dest] = value
+
+        if event_data.get("comments"):
+            ap_object["commentsEnabled"] = True
+
+        extra = event_data.get("extra")
+
+        if extra:
+            status = extra.get("status")
+            if status:
+                ap_object["ical:status"] = status.upper()
+
+            website = extra.get("website")
+            if website:
+                attachment.append({
+                    "href": website,
+                    "mediaType": "text/html",
+                    "name": "Website",
+                    "type": "Link"
+                })
+
+            accessibility = extra.get("accessibility")
+            if accessibility:
+                wheelchair = accessibility.get("wheelchair")
+                if wheelchair:
+                    if wheelchair == "full":
+                        ap_wc_value = "fully"
+                    elif wheelchair == "partial":
+                        ap_wc_value = "partially"
+                    elif wheelchair == "no":
+                        ap_wc_value = "no"
+                    else:
+                        log.error(f"unexpected wheelchair value: {wheelchair}")
+                        ap_wc_value = None
+                    if ap_wc_value is not None:
+                        attachment.append({
+                            "propertyID": "mz:accessibility:wheelchairAccessible",
+                            "type": "PropertyValue",
+                            "value": ap_wc_value
+                        })
+
+        activity = self.apg.create_activity(
+            "Create" if is_new else "Update", url_actor, ap_object, activity_id=url_item
+        )
+        activity["@context"].append(AP_EVENTS_CONTEXT)
+        return activity
+
+    async def ap_item_2_event_data(self, ap_item: dict) -> dict:
+        """Convert AP activity or object to event data
+
+        @param ap_item: ActivityPub item to convert
+            Can be either an activity of an object
+        @return: AP Item's Object and event data
+        @raise exceptions.DataError: something is invalid in the AP item
+        """
+        is_activity = self.apg.is_activity(ap_item)
+        if is_activity:
+            ap_object = await self.apg.ap_get_object(ap_item, "object")
+            if not ap_object:
+                log.warning(f'No "object" found in AP item {ap_item!r}')
+                raise exceptions.DataError
+        else:
+            ap_object = ap_item
+
+        # id
+        if "_repeated" in ap_item:
+            # if the event is repeated, we use the original one ID
+            repeated_uri = ap_item["_repeated"]["uri"]
+            parsed_uri = uri.parse_xmpp_uri(repeated_uri)
+            object_id = parsed_uri["item"]
+        else:
+            object_id = ap_object.get("id")
+            if not object_id:
+                raise exceptions.DataError('"id" is missing in AP object')
+
+        if ap_item["type"] != TYPE_EVENT:
+            raise exceptions.DataError("AP Object is not an event")
+
+        # author
+        actor = await self.apg.ap_get_sender_actor(ap_object)
+
+        account = await self.apg.get_ap_account_from_id(actor)
+        author_jid = self.apg.get_local_jid_from_account(account).full()
+
+        # name, start, end
+        event_data = {
+            "id": object_id,
+            "name": {"": ap_object.get("name") or "unnamed"},
+            "start": date_utils.date_parse(ap_object["startTime"]),
+            "end": date_utils.date_parse(ap_object["endTime"]),
+        }
+
+        # attachments/extra
+        event_data["extra"] = extra = {}
+        attachments = ap_object.get("attachment") or []
+        for attachment in attachments:
+            name = attachment.get("name")
+            if name == "Banner":
+                try:
+                    url = attachment["url"]
+                except KeyError:
+                    log.warning(f"invalid attachment: {attachment}")
+                    continue
+                event_data["head-picture"] = {"sources": [{"url": url}]}
+            elif name == "Website":
+                try:
+                    url = attachment["href"]
+                except KeyError:
+                    log.warning(f"invalid attachment: {attachment}")
+                    continue
+                extra["website"] = url
+            else:
+                log.debug(f"unmanaged attachment: {attachment}")
+
+        # description
+        content = ap_object.get("content")
+        if content:
+            event_data["descriptions"] = [{
+                "type": "xhtml",
+                "description": content
+            }]
+
+        # categories
+        tags = ap_object.get("tag")
+        if tags:
+            categories = event_data["categories"] = []
+            for tag in tags:
+                if tag.get("type") == "Hashtag":
+                    try:
+                        term = tag["name"][1:]
+                    except KeyError:
+                        log.warning(f"invalid tag: {tag}")
+                        continue
+                    categories.append({"term": term})
+
+        #location
+        ap_location = ap_object.get("location")
+        if ap_location:
+            location = {}
+            for source, dest in (
+                ("name", "description"),
+                ("latitude", "lat"),
+                ("longitude", "lon"),
+            ):
+                value = ap_location.get(source)
+                if value is not None:
+                    location[dest] = value
+            address = ap_location.get("address")
+            if address:
+                for source, dest in (
+                    ("addressCountry", "country"),
+                    ("addressLocality", "locality"),
+                    ("addressRegion", "region"),
+                    ("postalCode", "postalcode"),
+                    ("streetAddress", "street"),
+                ):
+                    value = address.get(source)
+                    if value is not None:
+                        location[dest] = value
+            if location:
+                event_data["locations"] = [location]
+
+        # rsvp
+        # So far Mobilizon seems to only handle participate/don't participate, thus we use
+        # a simple "yes"/"no" form.
+        rsvp_data = {"fields": []}
+        event_data["rsvp"] = [rsvp_data]
+        rsvp_data["fields"].append({
+            "type": "list-single",
+            "name": "attending",
+            "label": "Attending",
+            "options": [
+                {"label": "yes", "value": "yes"},
+                {"label": "no", "value": "no"}
+            ],
+            "required": True
+        })
+
+        # comments
+
+        if ap_object.get("commentsEnabled"):
+            __, comments_node = await self.apg.get_comments_nodes(object_id, None)
+            event_data["comments"] = {
+                "service": author_jid,
+                "node": comments_node,
+            }
+
+        # extra
+        # part of extra come from "attachment" above
+
+        status = ap_object.get("ical:status")
+        if status is None:
+            pass
+        elif status in ("CONFIRMED", "CANCELLED", "TENTATIVE"):
+            extra["status"] = status.lower()
+        else:
+            log.warning(f"unknown event status: {status}")
+
+        return event_data
+
+    async def ap_item_2_event_data_and_elt(
+        self,
+        ap_item: dict
+    ) -> Tuple[dict, domish.Element]:
+        """Convert AP item to parsed event data and corresponding item element"""
+        event_data = await self.ap_item_2_event_data(ap_item)
+        event_elt = self._events.event_data_2_event_elt(event_data)
+        item_elt = domish.Element((None, "item"))
+        item_elt["id"] = event_data["id"]
+        item_elt.addChild(event_elt)
+        return event_data, item_elt
+
+    async def ap_item_2_event_elt(self, ap_item: dict) -> domish.Element:
+        """Convert AP item to XMPP item element"""
+        __, item_elt = await self.ap_item_2_event_data_and_elt(ap_item)
+        return item_elt

libervia/backend/plugins/plugin_comp_ap_gateway/pubsub_service.py

@@ -0,0 +1,570 @@
+#!/usr/bin/env python3
+
+# Libervia ActivityPub Gateway
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import Optional, Tuple, List, Dict, Any, Union
+from urllib.parse import urlparse
+from pathlib import Path
+from base64 import b64encode
+import tempfile
+
+from twisted.internet import defer, threads
+from twisted.words.protocols.jabber import jid, error
+from twisted.words.xish import domish
+from wokkel import rsm, pubsub, disco
+
+from libervia.backend.core.i18n import _
+from libervia.backend.core import exceptions
+from libervia.backend.core.core_types import SatXMPPEntity
+from libervia.backend.core.log import getLogger
+from libervia.backend.core.constants import Const as C
+from libervia.backend.tools import image
+from libervia.backend.tools.utils import ensure_deferred
+from libervia.backend.tools.web import download_file
+from libervia.backend.memory.sqla_mapping import PubsubSub, SubscriptionState
+
+from .constants import (
+    TYPE_ACTOR,
+    ST_AVATAR,
+    MAX_AVATAR_SIZE
+)
+
+
+log = getLogger(__name__)
+
+# all nodes have the same config
+NODE_CONFIG = [
+    {"var": "pubsub#persist_items", "type": "boolean", "value": True},
+    {"var": "pubsub#max_items", "value": "max"},
+    {"var": "pubsub#access_model", "type": "list-single", "value": "open"},
+    {"var": "pubsub#publish_model", "type": "list-single", "value": "open"},
+
+]
+
+NODE_CONFIG_VALUES = {c["var"]: c["value"] for c in NODE_CONFIG}
+NODE_OPTIONS = {c["var"]: {} for c in NODE_CONFIG}
+for c in NODE_CONFIG:
+    NODE_OPTIONS[c["var"]].update({k:v for k,v in c.items() if k not in ("var", "value")})
+
+
+class APPubsubService(rsm.PubSubService):
+    """Pubsub service for XMPP requests"""
+
+    def __init__(self, apg):
+        super(APPubsubService, self).__init__()
+        self.host = apg.host
+        self.apg = apg
+        self.discoIdentity = {
+            "category": "pubsub",
+            "type": "service",
+            "name": "Libervia ActivityPub Gateway",
+        }
+
+    async def get_ap_actor_ids_and_inbox(
+        self,
+        requestor: jid.JID,
+        recipient: jid.JID,
+    ) -> Tuple[str, str, str]:
+        """Get AP actor IDs from requestor and destinee JIDs
+
+        @param requestor: XMPP entity doing a request to an AP actor via the gateway
+        @param recipient: JID mapping an AP actor via the gateway
+        @return: requestor actor ID, recipient actor ID and recipient inbox
+        @raise error.StanzaError: "item-not-found" is raised if not user part is specified
+            in requestor
+        """
+        if not recipient.user:
+            raise error.StanzaError(
+                "item-not-found",
+                text="No user part specified"
+            )
+        requestor_actor_id = self.apg.build_apurl(TYPE_ACTOR, requestor.userhost())
+        recipient_account = self.apg._e.unescape(recipient.user)
+        recipient_actor_id = await self.apg.get_ap_actor_id_from_account(recipient_account)
+        inbox = await self.apg.get_ap_inbox_from_id(recipient_actor_id, use_shared=False)
+        return requestor_actor_id, recipient_actor_id, inbox
+
+
+    @ensure_deferred
+    async def publish(self, requestor, service, nodeIdentifier, items):
+        if self.apg.local_only and not self.apg.is_local(requestor):
+            raise error.StanzaError(
+                "forbidden",
+                "Only local users can publish on this gateway."
+            )
+        if not service.user:
+            raise error.StanzaError(
+                "bad-request",
+                "You must specify an ActivityPub actor account in JID user part."
+            )
+        ap_account = self.apg._e.unescape(service.user)
+        if ap_account.count("@") != 1:
+            raise error.StanzaError(
+                "bad-request",
+                f"{ap_account!r} is not a valid ActivityPub actor account."
+            )
+
+        client = self.apg.client.get_virtual_client(requestor)
+        if self.apg._pa.is_attachment_node(nodeIdentifier):
+            await self.apg.convert_and_post_attachments(
+                client, ap_account, service, nodeIdentifier, items, publisher=requestor
+            )
+        else:
+            await self.apg.convert_and_post_items(
+                client, ap_account, service, nodeIdentifier, items
+            )
+            cached_node = await self.host.memory.storage.get_pubsub_node(
+                client, service, nodeIdentifier, with_subscriptions=True, create=True
+            )
+            await self.host.memory.storage.cache_pubsub_items(
+                client,
+                cached_node,
+                items
+            )
+            for subscription in cached_node.subscriptions:
+                if subscription.state != SubscriptionState.SUBSCRIBED:
+                    continue
+                self.notifyPublish(
+                    service,
+                    nodeIdentifier,
+                    [(subscription.subscriber, None, items)]
+                )
+
+    async def ap_following_2_elt(self, ap_item: dict) -> domish.Element:
+        """Convert actor ID from following collection to XMPP item"""
+        actor_id = ap_item["id"]
+        actor_jid = await self.apg.get_jid_from_id(actor_id)
+        subscription_elt = self.apg._pps.build_subscription_elt(
+            self.apg._m.namespace, actor_jid
+        )
+        item_elt = pubsub.Item(id=actor_id, payload=subscription_elt)
+        return item_elt
+
+    async def ap_follower_2_elt(self, ap_item: dict) -> domish.Element:
+        """Convert actor ID from followers collection to XMPP item"""
+        actor_id = ap_item["id"]
+        actor_jid = await self.apg.get_jid_from_id(actor_id)
+        subscriber_elt = self.apg._pps.build_subscriber_elt(actor_jid)
+        item_elt = pubsub.Item(id=actor_id, payload=subscriber_elt)
+        return item_elt
+
+    async def generate_v_card(self, ap_account: str) -> domish.Element:
+        """Generate vCard4 (XEP-0292) item element from ap_account's metadata"""
+        actor_data = await self.apg.get_ap_actor_data_from_account(ap_account)
+        identity_data = {}
+
+        summary = actor_data.get("summary")
+        # summary is HTML, we have to convert it to text
+        if summary:
+            identity_data["description"] = await self.apg._t.convert(
+                summary,
+                self.apg._t.SYNTAX_XHTML,
+                self.apg._t.SYNTAX_TEXT,
+                False,
+            )
+
+        for field in ("name", "preferredUsername"):
+            value = actor_data.get(field)
+            if value:
+                identity_data.setdefault("nicknames", []).append(value)
+        vcard_elt = self.apg._v.dict_2_v_card(identity_data)
+        item_elt = domish.Element((pubsub.NS_PUBSUB, "item"))
+        item_elt.addChild(vcard_elt)
+        item_elt["id"] = self.apg._p.ID_SINGLETON
+        return item_elt
+
+    async def get_avatar_data(
+        self,
+        client: SatXMPPEntity,
+        ap_account: str
+    ) -> Dict[str, Any]:
+        """Retrieve actor's avatar if any, cache it and file actor_data
+
+        ``cache_uid``, `path``` and ``media_type`` keys are always files
+        ``base64`` key is only filled if the file was not already in cache
+        """
+        actor_data = await self.apg.get_ap_actor_data_from_account(ap_account)
+
+        for icon in await self.apg.ap_get_list(actor_data, "icon"):
+            url = icon.get("url")
+            if icon["type"] != "Image" or not url:
+                continue
+            parsed_url = urlparse(url)
+            if not parsed_url.scheme in ("http", "https"):
+                log.warning(f"unexpected URL scheme: {url!r}")
+                continue
+            filename = Path(parsed_url.path).name
+            if not filename:
+                log.warning(f"ignoring URL with invald path: {url!r}")
+                continue
+            break
+        else:
+            raise error.StanzaError("item-not-found")
+
+        key = f"{ST_AVATAR}{url}"
+        cache_uid = await client._ap_storage.get(key)
+
+        if cache_uid is None:
+            cache = None
+        else:
+            cache = self.apg.host.common_cache.get_metadata(cache_uid)
+
+        if cache is None:
+            with tempfile.TemporaryDirectory() as dir_name:
+                dest_path = Path(dir_name, filename)
+                await download_file(url, dest_path, max_size=MAX_AVATAR_SIZE)
+                avatar_data = {
+                    "path": dest_path,
+                    "filename": filename,
+                    'media_type': image.guess_type(dest_path),
+                }
+
+                await self.apg._i.cache_avatar(
+                    self.apg.IMPORT_NAME,
+                    avatar_data
+                )
+        else:
+            avatar_data = {
+            "cache_uid": cache["uid"],
+            "path": cache["path"],
+            "media_type": cache["mime_type"]
+        }
+
+        return avatar_data
+
+    async def generate_avatar_metadata(
+        self,
+        client: SatXMPPEntity,
+        ap_account: str
+    ) -> domish.Element:
+        """Generate the metadata element for user avatar
+
+        @raise StanzaError("item-not-found"): no avatar is present in actor data (in
+            ``icon`` field)
+        """
+        avatar_data = await self.get_avatar_data(client, ap_account)
+        return self.apg._a.build_item_metadata_elt(avatar_data)
+
+    def _blocking_b_6_4_encode_avatar(self, avatar_data: Dict[str, Any]) -> None:
+        with avatar_data["path"].open("rb") as f:
+            avatar_data["base64"] = b64encode(f.read()).decode()
+
+    async def generate_avatar_data(
+        self,
+        client: SatXMPPEntity,
+        ap_account: str,
+        itemIdentifiers: Optional[List[str]],
+    ) -> domish.Element:
+        """Generate the data element for user avatar
+
+        @raise StanzaError("item-not-found"): no avatar cached with requested ID
+        """
+        if not itemIdentifiers:
+            avatar_data = await self.get_avatar_data(client, ap_account)
+            if "base64" not in avatar_data:
+                await threads.deferToThread(self._blocking_b_6_4_encode_avatar, avatar_data)
+        else:
+            if len(itemIdentifiers) > 1:
+                # only a single item ID is supported
+                raise error.StanzaError("item-not-found")
+            item_id = itemIdentifiers[0]
+            # just to be sure that that we don't have an empty string
+            assert item_id
+            cache_data = self.apg.host.common_cache.get_metadata(item_id)
+            if cache_data is None:
+                raise error.StanzaError("item-not-found")
+            avatar_data = {
+                "cache_uid": item_id,
+                "path": cache_data["path"]
+            }
+            await threads.deferToThread(self._blocking_b_6_4_encode_avatar, avatar_data)
+
+        return self.apg._a.build_item_data_elt(avatar_data)
+
+    @ensure_deferred
+    async def items(
+        self,
+        requestor: jid.JID,
+        service: jid.JID,
+        node: str,
+        maxItems: Optional[int],
+        itemIdentifiers: Optional[List[str]],
+        rsm_req: Optional[rsm.RSMRequest]
+    ) -> Tuple[List[domish.Element], Optional[rsm.RSMResponse]]:
+        if not service.user:
+            return [], None
+        ap_account = self.host.plugins["XEP-0106"].unescape(service.user)
+        if ap_account.count("@") != 1:
+            log.warning(f"Invalid AP account used by {requestor}: {ap_account!r}")
+            return [], None
+
+        # cached_node may be pre-filled with some nodes (e.g. attachments nodes),
+        # otherwise it is filled when suitable
+        cached_node = None
+        client = self.apg.client
+        kwargs = {}
+
+        if node == self.apg._pps.subscriptions_node:
+            collection_name = "following"
+            parser = self.ap_following_2_elt
+            kwargs["only_ids"] = True
+            use_cache = False
+        elif node.startswith(self.apg._pps.subscribers_node_prefix):
+            collection_name = "followers"
+            parser = self.ap_follower_2_elt
+            kwargs["only_ids"] = True
+            use_cache = False
+        elif node == self.apg._v.node:
+            # vCard4 request
+            item_elt = await self.generate_v_card(ap_account)
+            return [item_elt], None
+        elif node == self.apg._a.namespace_metadata:
+            item_elt = await self.generate_avatar_metadata(self.apg.client, ap_account)
+            return [item_elt], None
+        elif node == self.apg._a.namespace_data:
+            item_elt = await self.generate_avatar_data(
+                self.apg.client, ap_account, itemIdentifiers
+            )
+            return [item_elt], None
+        elif self.apg._pa.is_attachment_node(node):
+            use_cache = True
+            # we check cache here because we emit an item-not-found error if the node is
+            # not in cache, as we are not dealing with real AP items
+            cached_node = await self.host.memory.storage.get_pubsub_node(
+                client, service, node
+            )
+            if cached_node is None:
+                raise error.StanzaError("item-not-found")
+        else:
+            if node.startswith(self.apg._m.namespace):
+                parser = self.apg.ap_item_2_mb_elt
+            elif node.startswith(self.apg._events.namespace):
+                parser = self.apg.ap_events.ap_item_2_event_elt
+            else:
+                raise error.StanzaError(
+                    "feature-not-implemented",
+                    text=f"AP Gateway {C.APP_VERSION} only supports "
+                    f"{self.apg._m.namespace} node for now"
+                )
+            collection_name = "outbox"
+            use_cache = True
+
+        if use_cache:
+            if cached_node is None:
+                cached_node = await self.host.memory.storage.get_pubsub_node(
+                    client, service, node
+                )
+            # TODO: check if node is synchronised
+            if cached_node is not None:
+                # the node is cached, we return items from cache
+                log.debug(f"node {node!r} from {service} is in cache")
+                pubsub_items, metadata = await self.apg._c.get_items_from_cache(
+                    client, cached_node, maxItems, itemIdentifiers, rsm_request=rsm_req
+                )
+                try:
+                    rsm_resp = rsm.RSMResponse(**metadata["rsm"])
+                except KeyError:
+                    rsm_resp = None
+                return [i.data for i in pubsub_items], rsm_resp
+
+        if itemIdentifiers:
+            items = []
+            for item_id in itemIdentifiers:
+                item_data = await self.apg.ap_get(item_id)
+                item_elt = await parser(item_data)
+                items.append(item_elt)
+            return items, None
+        else:
+            if rsm_req is None:
+                if maxItems is None:
+                    maxItems = 20
+                kwargs.update({
+                    "max_items": maxItems,
+                    "chronological_pagination": False,
+                })
+            else:
+                if len(
+                    [v for v in (rsm_req.after, rsm_req.before, rsm_req.index)
+                     if v is not None]
+                ) > 1:
+                    raise error.StanzaError(
+                        "bad-request",
+                        text="You can't use after, before and index at the same time"
+                    )
+                kwargs.update({"max_items": rsm_req.max})
+                if rsm_req.after is not None:
+                    kwargs["after_id"] = rsm_req.after
+                elif rsm_req.before is not None:
+                    kwargs["chronological_pagination"] = False
+                    if rsm_req.before != "":
+                        kwargs["after_id"] = rsm_req.before
+                elif rsm_req.index is not None:
+                    kwargs["start_index"] = rsm_req.index
+
+            log.info(
+                f"No cache found for node {node} at {service} (AP account {ap_account}), "
+                "using Collection Paging to RSM translation"
+            )
+            if self.apg._m.is_comment_node(node):
+                parent_item = self.apg._m.get_parent_item(node)
+                try:
+                    parent_data = await self.apg.ap_get(parent_item)
+                    collection = await self.apg.ap_get_object(
+                        parent_data.get("object", {}),
+                        "replies"
+                    )
+                except Exception as e:
+                    raise error.StanzaError(
+                        "item-not-found",
+                        text=e
+                    )
+            else:
+                actor_data = await self.apg.get_ap_actor_data_from_account(ap_account)
+                collection = await self.apg.ap_get_object(actor_data, collection_name)
+            if not collection:
+                raise error.StanzaError(
+                    "item-not-found",
+                    text=f"No collection found for node {node!r} (account: {ap_account})"
+                )
+
+            kwargs["parser"] = parser
+            return await self.apg.get_ap_items(collection, **kwargs)
+
+    @ensure_deferred
+    async def retract(self, requestor, service, nodeIdentifier, itemIdentifiers):
+        raise error.StanzaError("forbidden")
+
+    @ensure_deferred
+    async def subscribe(self, requestor, service, nodeIdentifier, subscriber):
+        # TODO: handle comments nodes
+        client = self.apg.client
+        # we use PENDING state for microblog, it will be set to SUBSCRIBED once the Follow
+        # is accepted. Other nodes are directly set to subscribed, their subscriptions
+        # being internal.
+        if nodeIdentifier == self.apg._m.namespace:
+            sub_state = SubscriptionState.PENDING
+        else:
+            sub_state = SubscriptionState.SUBSCRIBED
+        node = await self.host.memory.storage.get_pubsub_node(
+            client, service, nodeIdentifier, with_subscriptions=True
+        )
+        if node is None:
+            node = await self.host.memory.storage.set_pubsub_node(
+                client,
+                service,
+                nodeIdentifier,
+            )
+            subscription = None
+        else:
+            try:
+                subscription = next(
+                    s for s in node.subscriptions
+                    if s.subscriber == requestor.userhostJID()
+                )
+            except StopIteration:
+                subscription = None
+
+        if subscription is None:
+            subscription = PubsubSub(
+                subscriber=requestor.userhostJID(),
+                state=sub_state
+            )
+            node.subscriptions.append(subscription)
+            await self.host.memory.storage.add(node)
+        else:
+            if subscription.state is None:
+                subscription.state = sub_state
+                await self.host.memory.storage.add(node)
+            elif subscription.state == SubscriptionState.SUBSCRIBED:
+                log.info(
+                    f"{requestor.userhostJID()} has already a subscription to {node!r} "
+                    f"at {service}. Doing the request anyway."
+                )
+            elif subscription.state == SubscriptionState.PENDING:
+                log.info(
+                    f"{requestor.userhostJID()} has already a pending subscription to "
+                    f"{node!r} at {service}. Doing the request anyway."
+                )
+                if sub_state != SubscriptionState.PENDING:
+                    subscription.state = sub_state
+                    await self.host.memory.storage.add(node)
+            else:
+                raise exceptions.InternalError(
+                    f"unmanaged subscription state: {subscription.state}"
+                )
+
+        if nodeIdentifier in (self.apg._m.namespace, self.apg._events.namespace):
+            # if we subscribe to microblog or events node, we follow the corresponding
+            # account
+            req_actor_id, recip_actor_id, inbox = await self.get_ap_actor_ids_and_inbox(
+                requestor, service
+            )
+
+            data = self.apg.create_activity("Follow", req_actor_id, recip_actor_id)
+
+            resp = await self.apg.sign_and_post(inbox, req_actor_id, data)
+            if resp.code >= 300:
+                text = await resp.text()
+                raise error.StanzaError("service-unavailable", text=text)
+        return pubsub.Subscription(nodeIdentifier, requestor, "subscribed")
+
+    @ensure_deferred
+    async def unsubscribe(self, requestor, service, nodeIdentifier, subscriber):
+        req_actor_id, recip_actor_id, inbox = await self.get_ap_actor_ids_and_inbox(
+            requestor, service
+        )
+        data = self.apg.create_activity(
+            "Undo",
+            req_actor_id,
+            self.apg.create_activity(
+                "Follow",
+                req_actor_id,
+                recip_actor_id
+            )
+        )
+
+        resp = await self.apg.sign_and_post(inbox, req_actor_id, data)
+        if resp.code >= 300:
+            text = await resp.text()
+            raise error.StanzaError("service-unavailable", text=text)
+
+    def getConfigurationOptions(self):
+        return NODE_OPTIONS
+
+    def getConfiguration(
+        self,
+        requestor: jid.JID,
+        service: jid.JID,
+        nodeIdentifier: str
+    ) -> defer.Deferred:
+        return defer.succeed(NODE_CONFIG_VALUES)
+
+    def getNodeInfo(
+        self,
+        requestor: jid.JID,
+        service: jid.JID,
+        nodeIdentifier: str,
+        pep: bool = False,
+        recipient: Optional[jid.JID] = None
+    ) -> Optional[dict]:
+        if not nodeIdentifier:
+            return None
+        info = {
+            "type": "leaf",
+            "meta-data": NODE_CONFIG
+        }
+        return info

libervia/backend/plugins/plugin_comp_ap_gateway/regex.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+# Libervia ActivityPub Gateway
+# Copyright (C) 2009-2022 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Various Regular Expression for AP gateway"""
+
+import re
+
+## "Signature" header parsing
+
+# those expression have been generated with abnf-to-regex
+# (https://github.com/aas-core-works/abnf-to-regexp)
+
+# the base RFC 7320 ABNF rules come from https://github.com/EricGT/ABNF
+
+# here is the ABNF file used:
+# ---
+# BWS = OWS
+# OWS = *( SP / HTAB )
+# tchar = "!" / "#" / "$" / "%" / "&" / "`" / "*" / "+" / "-" / "." / "^" / "_" / "\'" / "|" / "~" / DIGIT / ALPHA
+# token = 1*tchar
+# sig-param = token BWS "=" BWS ( token / quoted-string )
+# quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE
+# qdtext = HTAB / SP / "!" / %x23-5B ; '#'-'['
+#  / %x5D-7E ; ']'-'~'
+#  / obs-text
+# quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text )
+# obs-text = %x80-FF
+# ---
+
+ows = '[ \t]*'
+bws = f'{ows}'
+obs_text = '[\\x80-\\xff]'
+qdtext = f'([\t !#-\\[\\]-~]|{obs_text})'
+quoted_pair = f'\\\\([\t !-~]|{obs_text})'
+quoted_string = f'"({qdtext}|{quoted_pair})*"'
+tchar = "([!#$%&`*+\\-.^_]|\\\\'|[|~0-9a-zA-Z])"
+token = f'({tchar})+'
+RE_SIG_PARAM = re.compile(
+    f'(?P<key>{token}{bws})={bws}'
+    f'((?P<uq_value>{token})|(?P<quoted_value>{quoted_string}))'
+)
+
+
+## Account/Mention
+
+# FIXME: naive regex, should be approved following webfinger, but popular implementations
+#   such as Mastodon use a very restricted subset
+RE_ACCOUNT = re.compile(r"[a-zA-Z0-9._-]+@[a-zA-Z0-9-]+.[a-zA-Z0-9-]+")
+RE_MENTION = re.compile(rf"(?<!\w)@{RE_ACCOUNT.pattern}\b")