refactoring: renamed sat package to libervia.backend

libervia/backend/plugins/plugin_adhoc_dbus.py

@@ -0,0 +1,478 @@
+#!/usr/bin/env python3
+
+
+# SAT plugin for adding D-Bus to Ad-Hoc Commands
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from libervia.backend.core.i18n import D_, _
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.log import getLogger
+
+log = getLogger(__name__)
+from twisted.internet import defer
+from twisted.words.protocols.jabber import jid
+from wokkel import data_form
+
+try:
+    from lxml import etree
+except ImportError:
+    etree = None
+    log.warning("Missing module lxml, please download/install it from http://lxml.de/ ."
+                "Auto D-Bus discovery will be disabled")
+from collections import OrderedDict
+import os.path
+import uuid
+try:
+    import dbus
+    from dbus.mainloop.glib import DBusGMainLoop
+except ImportError:
+    dbus = None
+    log.warning("Missing module dbus, please download/install it, "
+                "auto D-Bus discovery will be disabled")
+
+else:
+    DBusGMainLoop(set_as_default=True)
+
+NS_MEDIA_PLAYER = "org.libervia.mediaplayer"
+FD_NAME = "org.freedesktop.DBus"
+FD_PATH = "/org/freedekstop/DBus"
+INTROSPECT_IFACE = "org.freedesktop.DBus.Introspectable"
+MPRIS_PREFIX = "org.mpris.MediaPlayer2"
+CMD_GO_BACK = "GoBack"
+CMD_GO_FWD = "GoFW"
+SEEK_OFFSET = 5 * 1000 * 1000
+MPRIS_COMMANDS = ["org.mpris.MediaPlayer2.Player." + cmd for cmd in (
+    "Previous", CMD_GO_BACK, "PlayPause", CMD_GO_FWD, "Next")]
+MPRIS_PATH = "/org/mpris/MediaPlayer2"
+MPRIS_PROPERTIES = OrderedDict((
+    ("org.mpris.MediaPlayer2", (
+        "Identity",
+        )),
+    ("org.mpris.MediaPlayer2.Player", (
+        "Metadata",
+        "PlaybackStatus",
+        "Volume",
+        )),
+    ))
+MPRIS_METADATA_KEY = "Metadata"
+MPRIS_METADATA_MAP = OrderedDict((
+    ("xesam:title", "Title"),
+    ))
+
+INTROSPECT_METHOD = "Introspect"
+IGNORED_IFACES_START = (
+    "org.freedesktop",
+    "org.qtproject",
+    "org.kde.KMainWindow",
+)  # commands in interface starting with these values will be ignored
+FLAG_LOOP = "LOOP"
+
+PLUGIN_INFO = {
+    C.PI_NAME: "Ad-Hoc Commands - D-Bus",
+    C.PI_IMPORT_NAME: "AD_HOC_DBUS",
+    C.PI_TYPE: "Misc",
+    C.PI_PROTOCOLS: [],
+    C.PI_DEPENDENCIES: ["XEP-0050"],
+    C.PI_MAIN: "AdHocDBus",
+    C.PI_HANDLER: "no",
+    C.PI_DESCRIPTION: _("""Add D-Bus management to Ad-Hoc commands"""),
+}
+
+
+class AdHocDBus(object):
+
+    def __init__(self, host):
+        log.info(_("plugin Ad-Hoc D-Bus initialization"))
+        self.host = host
+        if etree is not None:
+            host.bridge.add_method(
+                "ad_hoc_dbus_add_auto",
+                ".plugin",
+                in_sign="sasasasasasass",
+                out_sign="(sa(sss))",
+                method=self._ad_hoc_dbus_add_auto,
+                async_=True,
+            )
+        host.bridge.add_method(
+            "ad_hoc_remotes_get",
+            ".plugin",
+            in_sign="s",
+            out_sign="a(sss)",
+            method=self._ad_hoc_remotes_get,
+            async_=True,
+        )
+        self._c = host.plugins["XEP-0050"]
+        host.register_namespace("mediaplayer", NS_MEDIA_PLAYER)
+        if dbus is not None:
+            self.session_bus = dbus.SessionBus()
+            self.fd_object = self.session_bus.get_object(
+                FD_NAME, FD_PATH, introspect=False)
+
+    def profile_connected(self, client):
+        if dbus is not None:
+            self._c.add_ad_hoc_command(
+                client, self.local_media_cb, D_("Media Players"),
+                node=NS_MEDIA_PLAYER,
+                timeout=60*60*6  # 6 hours timeout, to avoid breaking remote
+                                 # in the middle of a movie
+            )
+
+    def _dbus_async_call(self, proxy, method, *args, **kwargs):
+        """ Call a DBus method asynchronously and return a deferred
+
+        @param proxy: DBus object proxy, as returner by get_object
+        @param method: name of the method to call
+        @param args: will be transmitted to the method
+        @param kwargs: will be transmetted to the method, except for the following poped
+                       values:
+                       - interface: name of the interface to use
+        @return: a deferred
+
+        """
+        d = defer.Deferred()
+        interface = kwargs.pop("interface", None)
+        kwargs["reply_handler"] = lambda ret=None: d.callback(ret)
+        kwargs["error_handler"] = d.errback
+        proxy.get_dbus_method(method, dbus_interface=interface)(*args, **kwargs)
+        return d
+
+    def _dbus_get_property(self, proxy, interface, name):
+        return self._dbus_async_call(
+            proxy, "Get", interface, name, interface="org.freedesktop.DBus.Properties")
+
+
+    def _dbus_list_names(self):
+        return self._dbus_async_call(self.fd_object, "ListNames")
+
+    def _dbus_introspect(self, proxy):
+        return self._dbus_async_call(proxy, INTROSPECT_METHOD, interface=INTROSPECT_IFACE)
+
+    def _accept_method(self, method):
+        """ Return True if we accept the method for a command
+        @param method: etree.Element
+        @return: True if the method is acceptable
+
+        """
+        if method.xpath(
+            "arg[@direction='in']"
+        ):  # we don't accept method with argument for the moment
+            return False
+        return True
+
+    @defer.inlineCallbacks
+    def _introspect(self, methods, bus_name, proxy):
+        log.debug("introspecting path [%s]" % proxy.object_path)
+        introspect_xml = yield self._dbus_introspect(proxy)
+        el = etree.fromstring(introspect_xml)
+        for node in el.iterchildren("node", "interface"):
+            if node.tag == "node":
+                new_path = os.path.join(proxy.object_path, node.get("name"))
+                new_proxy = self.session_bus.get_object(
+                    bus_name, new_path, introspect=False
+                )
+                yield self._introspect(methods, bus_name, new_proxy)
+            elif node.tag == "interface":
+                name = node.get("name")
+                if any(name.startswith(ignored) for ignored in IGNORED_IFACES_START):
+                    log.debug("interface [%s] is ignored" % name)
+                    continue
+                log.debug("introspecting interface [%s]" % name)
+                for method in node.iterchildren("method"):
+                    if self._accept_method(method):
+                        method_name = method.get("name")
+                        log.debug("method accepted: [%s]" % method_name)
+                        methods.add((proxy.object_path, name, method_name))
+
+    def _ad_hoc_dbus_add_auto(self, prog_name, allowed_jids, allowed_groups, allowed_magics,
+                          forbidden_jids, forbidden_groups, flags, profile_key):
+        client = self.host.get_client(profile_key)
+        return self.ad_hoc_dbus_add_auto(
+            client, prog_name, allowed_jids, allowed_groups, allowed_magics,
+            forbidden_jids, forbidden_groups, flags)
+
+    @defer.inlineCallbacks
+    def ad_hoc_dbus_add_auto(self, client, prog_name, allowed_jids=None, allowed_groups=None,
+                         allowed_magics=None, forbidden_jids=None, forbidden_groups=None,
+                         flags=None):
+        bus_names = yield self._dbus_list_names()
+        bus_names = [bus_name for bus_name in bus_names if "." + prog_name in bus_name]
+        if not bus_names:
+            log.info("Can't find any bus for [%s]" % prog_name)
+            defer.returnValue(("", []))
+        bus_names.sort()
+        for bus_name in bus_names:
+            if bus_name.endswith(prog_name):
+                break
+        log.info("bus name found: [%s]" % bus_name)
+        proxy = self.session_bus.get_object(bus_name, "/", introspect=False)
+        methods = set()
+
+        yield self._introspect(methods, bus_name, proxy)
+
+        if methods:
+            self._add_command(
+                client,
+                prog_name,
+                bus_name,
+                methods,
+                allowed_jids=allowed_jids,
+                allowed_groups=allowed_groups,
+                allowed_magics=allowed_magics,
+                forbidden_jids=forbidden_jids,
+                forbidden_groups=forbidden_groups,
+                flags=flags,
+            )
+
+        defer.returnValue((str(bus_name), methods))
+
+    def _add_command(self, client, adhoc_name, bus_name, methods, allowed_jids=None,
+                    allowed_groups=None, allowed_magics=None, forbidden_jids=None,
+                    forbidden_groups=None, flags=None):
+        if flags is None:
+            flags = set()
+
+        def d_bus_callback(client, command_elt, session_data, action, node):
+            actions = session_data.setdefault("actions", [])
+            names_map = session_data.setdefault("names_map", {})
+            actions.append(action)
+
+            if len(actions) == 1:
+                # it's our first request, we ask the desired new status
+                status = self._c.STATUS.EXECUTING
+                form = data_form.Form("form", title=_("Command selection"))
+                options = []
+                for path, iface, command in methods:
+                    label = command.rsplit(".", 1)[-1]
+                    name = str(uuid.uuid4())
+                    names_map[name] = (path, iface, command)
+                    options.append(data_form.Option(name, label))
+
+                field = data_form.Field(
+                    "list-single", "command", options=options, required=True
+                )
+                form.addField(field)
+
+                payload = form.toElement()
+                note = None
+
+            elif len(actions) == 2:
+                # we should have the answer here
+                try:
+                    x_elt = next(command_elt.elements(data_form.NS_X_DATA, "x"))
+                    answer_form = data_form.Form.fromElement(x_elt)
+                    command = answer_form["command"]
+                except (KeyError, StopIteration):
+                    raise self._c.AdHocError(self._c.ERROR.BAD_PAYLOAD)
+
+                if command not in names_map:
+                    raise self._c.AdHocError(self._c.ERROR.BAD_PAYLOAD)
+
+                path, iface, command = names_map[command]
+                proxy = self.session_bus.get_object(bus_name, path)
+
+                self._dbus_async_call(proxy, command, interface=iface)
+
+                # job done, we can end the session, except if we have FLAG_LOOP
+                if FLAG_LOOP in flags:
+                    # We have a loop, so we clear everything and we execute again the
+                    # command as we had a first call (command_elt is not used, so None
+                    # is OK)
+                    del actions[:]
+                    names_map.clear()
+                    return d_bus_callback(
+                        client, None, session_data, self._c.ACTION.EXECUTE, node
+                    )
+                form = data_form.Form("form", title=_("Updated"))
+                form.addField(data_form.Field("fixed", "Command sent"))
+                status = self._c.STATUS.COMPLETED
+                payload = None
+                note = (self._c.NOTE.INFO, _("Command sent"))
+            else:
+                raise self._c.AdHocError(self._c.ERROR.INTERNAL)
+
+            return (payload, status, None, note)
+
+        self._c.add_ad_hoc_command(
+            client,
+            d_bus_callback,
+            adhoc_name,
+            allowed_jids=allowed_jids,
+            allowed_groups=allowed_groups,
+            allowed_magics=allowed_magics,
+            forbidden_jids=forbidden_jids,
+            forbidden_groups=forbidden_groups,
+        )
+
+    ## Local media ##
+
+    def _ad_hoc_remotes_get(self, profile):
+        return self.ad_hoc_remotes_get(self.host.get_client(profile))
+
+    @defer.inlineCallbacks
+    def ad_hoc_remotes_get(self, client):
+        """Retrieve available remote media controlers in our devices
+        @return (list[tuple[unicode, unicode, unicode]]): list of devices with:
+            - entity full jid
+            - device name
+            - device label
+        """
+        found_data = yield defer.ensureDeferred(self.host.find_by_features(
+            client, [self.host.ns_map['commands']], service=False, roster=False,
+            own_jid=True, local_device=True))
+
+        remotes = []
+
+        for found in found_data:
+            for device_jid_s in found:
+                device_jid = jid.JID(device_jid_s)
+                cmd_list = yield self._c.list(client, device_jid)
+                for cmd in cmd_list:
+                    if cmd.nodeIdentifier == NS_MEDIA_PLAYER:
+                        try:
+                            result_elt = yield self._c.do(client, device_jid,
+                                                          NS_MEDIA_PLAYER, timeout=5)
+                            command_elt = self._c.get_command_elt(result_elt)
+                            form = data_form.findForm(command_elt, NS_MEDIA_PLAYER)
+                            if form is None:
+                                continue
+                            mp_options = form.fields['media_player'].options
+                            session_id = command_elt.getAttribute('sessionid')
+                            if mp_options and session_id:
+                                # we just want to discover player, so we cancel the
+                                # session
+                                self._c.do(client, device_jid, NS_MEDIA_PLAYER,
+                                           action=self._c.ACTION.CANCEL,
+                                           session_id=session_id)
+
+                            for opt in mp_options:
+                                remotes.append((device_jid_s,
+                                                opt.value,
+                                                opt.label or opt.value))
+                        except Exception as e:
+                            log.warning(_(
+                                "Can't retrieve remote controllers on {device_jid}: "
+                                "{reason}".format(device_jid=device_jid, reason=e)))
+                        break
+        defer.returnValue(remotes)
+
+    def do_mpris_command(self, proxy, command):
+        iface, command = command.rsplit(".", 1)
+        if command == CMD_GO_BACK:
+            command = 'Seek'
+            args = [-SEEK_OFFSET]
+        elif command == CMD_GO_FWD:
+            command = 'Seek'
+            args = [SEEK_OFFSET]
+        else:
+            args = []
+        return self._dbus_async_call(proxy, command, *args, interface=iface)
+
+    def add_mpris_metadata(self, form, metadata):
+        """Serialise MRPIS Metadata according to MPRIS_METADATA_MAP"""
+        for mpris_key, name in MPRIS_METADATA_MAP.items():
+            if mpris_key in metadata:
+                value = str(metadata[mpris_key])
+                form.addField(data_form.Field(fieldType="fixed",
+                                              var=name,
+                                              value=value))
+
+    @defer.inlineCallbacks
+    def local_media_cb(self, client, command_elt, session_data, action, node):
+        try:
+            x_elt = next(command_elt.elements(data_form.NS_X_DATA, "x"))
+            command_form = data_form.Form.fromElement(x_elt)
+        except StopIteration:
+            command_form = None
+
+        if command_form is None or len(command_form.fields) == 0:
+            # root request, we looks for media players
+            bus_names = yield self._dbus_list_names()
+            bus_names = [b for b in bus_names if b.startswith(MPRIS_PREFIX)]
+            if len(bus_names) == 0:
+                note = (self._c.NOTE.INFO, D_("No media player found."))
+                defer.returnValue((None, self._c.STATUS.COMPLETED, None, note))
+            options = []
+            status = self._c.STATUS.EXECUTING
+            form = data_form.Form("form", title=D_("Media Player Selection"),
+                                  formNamespace=NS_MEDIA_PLAYER)
+            for bus in bus_names:
+                player_name = bus[len(MPRIS_PREFIX)+1:]
+                if not player_name:
+                    log.warning(_("Ignoring MPRIS bus without suffix"))
+                    continue
+                options.append(data_form.Option(bus, player_name))
+            field = data_form.Field(
+                "list-single", "media_player", options=options, required=True
+            )
+            form.addField(field)
+            payload = form.toElement()
+            defer.returnValue((payload, status, None, None))
+        else:
+            # player request
+            try:
+                bus_name = command_form["media_player"]
+            except KeyError:
+                raise ValueError(_("missing media_player value"))
+
+            if not bus_name.startswith(MPRIS_PREFIX):
+                log.warning(_("Media player ad-hoc command trying to use non MPRIS bus. "
+                              "Hack attempt? Refused bus: {bus_name}").format(
+                              bus_name=bus_name))
+                note = (self._c.NOTE.ERROR, D_("Invalid player name."))
+                defer.returnValue((None, self._c.STATUS.COMPLETED, None, note))
+
+            try:
+                proxy = self.session_bus.get_object(bus_name, MPRIS_PATH)
+            except dbus.exceptions.DBusException as e:
+                log.warning(_("Can't get D-Bus proxy: {reason}").format(reason=e))
+                note = (self._c.NOTE.ERROR, D_("Media player is not available anymore"))
+                defer.returnValue((None, self._c.STATUS.COMPLETED, None, note))
+            try:
+                command = command_form["command"]
+            except KeyError:
+                pass
+            else:
+                yield self.do_mpris_command(proxy, command)
+
+            # we construct the remote control form
+            form = data_form.Form("form", title=D_("Media Player Selection"))
+            form.addField(data_form.Field(fieldType="hidden",
+                                          var="media_player",
+                                          value=bus_name))
+            for iface, properties_names in MPRIS_PROPERTIES.items():
+                for name in properties_names:
+                    try:
+                        value = yield self._dbus_get_property(proxy, iface, name)
+                    except Exception as e:
+                        log.warning(_("Can't retrieve attribute {name}: {reason}")
+                                    .format(name=name, reason=e))
+                        continue
+                    if name == MPRIS_METADATA_KEY:
+                        self.add_mpris_metadata(form, value)
+                    else:
+                        form.addField(data_form.Field(fieldType="fixed",
+                                                      var=name,
+                                                      value=str(value)))
+
+            commands = [data_form.Option(c, c.rsplit(".", 1)[1]) for c in MPRIS_COMMANDS]
+            form.addField(data_form.Field(fieldType="list-single",
+                                          var="command",
+                                          options=commands,
+                                          required=True))
+
+            payload = form.toElement()
+            status = self._c.STATUS.EXECUTING
+            defer.returnValue((payload, status, None, None))

libervia/backend/plugins/plugin_app_manager_docker/__init__.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+# SàT plugin to manage Docker
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from pathlib import Path
+from twisted.python.procutils import which
+from libervia.backend.core.i18n import _
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core import exceptions
+from libervia.backend.core.log import getLogger
+from libervia.backend.tools.common import async_process
+
+log = getLogger(__name__)
+
+
+PLUGIN_INFO = {
+    C.PI_NAME: "Docker Applications Manager",
+    C.PI_IMPORT_NAME: "APP_MANAGER_DOCKER",
+    C.PI_TYPE: C.PLUG_TYPE_MISC,
+    C.PI_MODES: C.PLUG_MODE_BOTH,
+    C.PI_DEPENDENCIES: ["APP_MANAGER"],
+    C.PI_MAIN: "AppManagerDocker",
+    C.PI_HANDLER: "no",
+    C.PI_DESCRIPTION: _(
+        """Applications Manager for Docker"""),
+}
+
+
+class AppManagerDocker:
+    name = "docker-compose"
+    discover_path = Path(__file__).parent
+
+    def __init__(self, host):
+        log.info(_("Docker App Manager initialization"))
+        try:
+            self.docker_compose_path = which('docker-compose')[0]
+        except IndexError:
+            raise exceptions.NotFound(
+                '"docker-compose" executable not found, Docker can\'t be used with '
+                'application manager')
+        self.host = host
+        self._am = host.plugins['APP_MANAGER']
+        self._am.register(self)
+
+    async def start(self, app_data: dict) -> None:
+        await self._am.start_common(app_data)
+        working_dir = app_data['_instance_dir_path']
+        try:
+            override = app_data['override']
+        except KeyError:
+            pass
+        else:
+            log.debug("writting override file")
+            override_path = working_dir / "docker-compose.override.yml"
+            with override_path.open("w") as f:
+                self._am.dump(override, f)
+        await async_process.run(
+            self.docker_compose_path,
+            "up",
+            "--detach",
+            path=str(working_dir),
+        )
+
+    async def stop(self, app_data: dict) -> None:
+        working_dir = app_data['_instance_dir_path']
+        await async_process.run(
+            self.docker_compose_path,
+            "down",
+            path=str(working_dir),
+        )
+
+    async def compute_expose(self, app_data: dict) -> dict:
+        working_dir = app_data['_instance_dir_path']
+        expose = app_data['expose']
+        ports = expose.get('ports', {})
+        for name, port_data in list(ports.items()):
+            try:
+                service = port_data['service']
+                private = port_data['private']
+                int(private)
+            except (KeyError, ValueError):
+                log.warning(
+                    f"invalid value found for {name!r} port in {app_data['_file_path']}")
+                continue
+            exposed_port = await async_process.run(
+                self.docker_compose_path,
+                "port",
+                service,
+                str(private),
+                path=str(working_dir),
+            )
+            exposed_port = exposed_port.decode().strip()
+            try:
+                addr, port = exposed_port.split(':')
+                int(port)
+            except ValueError:
+                log.warning(
+                    f"invalid exposed port for {name}, ignoring: {exposed_port!r}")
+                del ports[name]
+            else:
+                ports[name] = exposed_port

libervia/backend/plugins/plugin_app_manager_docker/sat_app_weblate.yaml

@@ -0,0 +1,37 @@
+type: docker-compose
+prepare:
+  git: https://github.com/WeblateOrg/docker-compose.git
+files:
+  settings-override.py:
+    content: |
+      USE_X_FORWARDED_HOST = True
+override:
+  version: "3"
+  services:
+    weblate:
+      ports:
+        - "8080"
+      environment:
+        WEBLATE_DEBUG: 0
+        WEBLATE_URL_PREFIX: !sat_param [url_prefix, /weblate]
+        WEBLATE_EMAIL_HOST: !sat_conf ["", "email_server"]
+        WEBLATE_EMAIL_HOST_USER: !sat_conf ["", "email_username"]
+        WEBLATE_EMAIL_HOST_PASSWORD: !sat_conf ["", "email_password"]
+        WEBLATE_SERVER_EMAIL: !sat_conf ["", "email_from", "weblate@example.com"]
+        WEBLATE_DEFAULT_FROM_EMAIL: !sat_conf ["", "email_from", "weblate@example.com"]
+        WEBLATE_SITE_DOMAIN: !sat_conf ["", "public_url"]
+        WEBLATE_ADMIN_PASSWORD: !sat_generate_pwd
+        WEBLATE_ADMIN_EMAIL: !sat_conf ["", "email_admins_list", "", "first"]
+        WEBLATE_ENABLE_HTTPS: !sat_conf ["", "weblate_enable_https", "1"]
+      volumes:
+        - ./settings-override.py:/app/data/settings-override.py:ro
+expose:
+  url_prefix: [override, services, weblate, environment, WEBLATE_URL_PREFIX]
+  front_url: !sat_param [front_url, /translate]
+  web_label: Translate
+  ports:
+    web:
+      service: weblate
+      private: 8080
+  passwords:
+    admin: [override, services, weblate, environment, WEBLATE_ADMIN_PASSWORD]

libervia/backend/plugins/plugin_blog_import.py

@@ -0,0 +1,323 @@
+#!/usr/bin/env python3
+
+
+# SàT plugin for import external blogs
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from libervia.backend.core.i18n import _
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.log import getLogger
+
+log = getLogger(__name__)
+from twisted.internet import defer
+from twisted.web import client as web_client
+from twisted.words.xish import domish
+from libervia.backend.core import exceptions
+from libervia.backend.tools import xml_tools
+import os
+import os.path
+import tempfile
+import urllib.parse
+import shortuuid
+
+
+PLUGIN_INFO = {
+    C.PI_NAME: "blog import",
+    C.PI_IMPORT_NAME: "BLOG_IMPORT",
+    C.PI_TYPE: (C.PLUG_TYPE_BLOG, C.PLUG_TYPE_IMPORT),
+    C.PI_DEPENDENCIES: ["IMPORT", "XEP-0060", "XEP-0277", "TEXT_SYNTAXES", "UPLOAD"],
+    C.PI_MAIN: "BlogImportPlugin",
+    C.PI_HANDLER: "no",
+    C.PI_DESCRIPTION: _(
+        """Blog import management:
+This plugin manage the different blog importers which can register to it, and handle generic importing tasks."""
+    ),
+}
+
+OPT_HOST = "host"
+OPT_UPLOAD_IMAGES = "upload_images"
+OPT_UPLOAD_IGNORE_HOST = "upload_ignore_host"
+OPT_IGNORE_TLS = "ignore_tls_errors"
+URL_REDIRECT_PREFIX = "url_redirect_"
+
+
+class BlogImportPlugin(object):
+    BOOL_OPTIONS = (OPT_UPLOAD_IMAGES, OPT_IGNORE_TLS)
+    JSON_OPTIONS = ()
+    OPT_DEFAULTS = {OPT_UPLOAD_IMAGES: True, OPT_IGNORE_TLS: False}
+
+    def __init__(self, host):
+        log.info(_("plugin Blog import initialization"))
+        self.host = host
+        self._u = host.plugins["UPLOAD"]
+        self._p = host.plugins["XEP-0060"]
+        self._m = host.plugins["XEP-0277"]
+        self._s = self.host.plugins["TEXT_SYNTAXES"]
+        host.plugins["IMPORT"].initialize(self, "blog")
+
+    def import_item(
+        self, client, item_import_data, session, options, return_data, service, node
+    ):
+        """import_item specialized for blog import
+
+        @param item_import_data(dict):
+            * mandatory keys:
+                'blog' (dict): microblog data of the blog post (cf. http://wiki.goffi.org/wiki/Bridge_API_-_Microblogging/en)
+                    the importer MUST NOT create node or call XEP-0277 plugin itself
+                    'comments*' key MUST NOT be used in this microblog_data, see bellow for comments
+                    It is recommanded to use a unique id in the "id" key which is constant per blog item,
+                    so if the import fail, a new import will overwrite the failed items and avoid duplicates.
+
+                'comments' (list[list[dict]],None): Dictionaries must have the same keys as main item (i.e. 'blog' and 'comments')
+                    a list of list is used because XEP-0277 can handler several comments nodes,
+                    but in most cases, there will we only one item it the first list (something like [[{comment1_data},{comment2_data}, ...]])
+                    blog['allow_comments'] must be True if there is any comment, and False (or not present) if comments are not allowed.
+                    If allow_comments is False and some comments are present, an exceptions.DataError will be raised
+            * optional keys:
+                'url' (unicode): former url of the post (only the path, without host part)
+                    if present the association to the new path will be displayed to user, so it can make redirections if necessary
+        @param options(dict, None): Below are the generic options,
+            blog importer can have specific ones. All options have unicode values
+            generic options:
+                - OPT_HOST (unicode): original host
+                - OPT_UPLOAD_IMAGES (bool): upload images to XMPP server if True
+                    see OPT_UPLOAD_IGNORE_HOST.
+                    Default: True
+                - OPT_UPLOAD_IGNORE_HOST (unicode): don't upload images from this host
+                - OPT_IGNORE_TLS (bool): ignore TLS error for image upload.
+                    Default: False
+        @param return_data(dict): will contain link between former posts and new items
+
+        """
+        mb_data = item_import_data["blog"]
+        try:
+            item_id = mb_data["id"]
+        except KeyError:
+            item_id = mb_data["id"] = str(shortuuid.uuid())
+
+        try:
+            # we keep the link between old url and new blog item
+            # so the user can redirect its former blog urls
+            old_uri = item_import_data["url"]
+        except KeyError:
+            pass
+        else:
+            new_uri = return_data[URL_REDIRECT_PREFIX + old_uri] = self._p.get_node_uri(
+                service if service is not None else client.jid.userhostJID(),
+                node or self._m.namespace,
+                item_id,
+            )
+            log.info("url link from {old} to {new}".format(old=old_uri, new=new_uri))
+
+        return mb_data
+
+    @defer.inlineCallbacks
+    def import_sub_items(self, client, item_import_data, mb_data, session, options):
+        # comments data
+        if len(item_import_data["comments"]) != 1:
+            raise NotImplementedError("can't manage multiple comment links")
+        allow_comments = C.bool(mb_data.get("allow_comments", C.BOOL_FALSE))
+        if allow_comments:
+            comments_service = yield self._m.get_comments_service(client)
+            comments_node = self._m.get_comments_node(mb_data["id"])
+            mb_data["comments_service"] = comments_service.full()
+            mb_data["comments_node"] = comments_node
+            recurse_kwargs = {
+                "items_import_data": item_import_data["comments"][0],
+                "service": comments_service,
+                "node": comments_node,
+            }
+            defer.returnValue(recurse_kwargs)
+        else:
+            if item_import_data["comments"][0]:
+                raise exceptions.DataError(
+                    "allow_comments set to False, but comments are there"
+                )
+            defer.returnValue(None)
+
+    def publish_item(self, client, mb_data, service, node, session):
+        log.debug(
+            "uploading item [{id}]: {title}".format(
+                id=mb_data["id"], title=mb_data.get("title", "")
+            )
+        )
+        return self._m.send(client, mb_data, service, node)
+
+    @defer.inlineCallbacks
+    def item_filters(self, client, mb_data, session, options):
+        """Apply filters according to options
+
+        modify mb_data in place
+        @param posts_data(list[dict]): data as returned by importer callback
+        @param options(dict): dict as given in [blogImport]
+        """
+        # FIXME: blog filters don't work on text content
+        # TODO: text => XHTML conversion should handler links with <a/>
+        #       filters can then be used by converting text to XHTML
+        if not options:
+            return
+
+        # we want only XHTML content
+        for prefix in (
+            "content",
+        ):  # a tuple is use, if title need to be added in the future
+            try:
+                rich = mb_data["{}_rich".format(prefix)]
+            except KeyError:
+                pass
+            else:
+                if "{}_xhtml".format(prefix) in mb_data:
+                    raise exceptions.DataError(
+                        "importer gave {prefix}_rich and {prefix}_xhtml at the same time, this is not allowed".format(
+                            prefix=prefix
+                        )
+                    )
+                # we convert rich syntax to XHTML here, so we can handle filters easily
+                converted = yield self._s.convert(
+                    rich, self._s.get_current_syntax(client.profile), safe=False
+                )
+                mb_data["{}_xhtml".format(prefix)] = converted
+                del mb_data["{}_rich".format(prefix)]
+
+            try:
+                mb_data["txt"]
+            except KeyError:
+                pass
+            else:
+                if "{}_xhtml".format(prefix) in mb_data:
+                    log.warning(
+                        "{prefix}_text will be replaced by converted {prefix}_xhtml, so filters can be handled".format(
+                            prefix=prefix
+                        )
+                    )
+                    del mb_data["{}_text".format(prefix)]
+                else:
+                    log.warning(
+                        "importer gave a text {prefix}, blog filters don't work on text {prefix}".format(
+                            prefix=prefix
+                        )
+                    )
+                    return
+
+        # at this point, we have only XHTML version of content
+        try:
+            top_elt = xml_tools.ElementParser()(
+                mb_data["content_xhtml"], namespace=C.NS_XHTML
+            )
+        except domish.ParserError:
+            # we clean the xml and try again our luck
+            cleaned = yield self._s.clean_xhtml(mb_data["content_xhtml"])
+            top_elt = xml_tools.ElementParser()(cleaned, namespace=C.NS_XHTML)
+        opt_host = options.get(OPT_HOST)
+        if opt_host:
+            # we normalise the domain
+            parsed_host = urllib.parse.urlsplit(opt_host)
+            opt_host = urllib.parse.urlunsplit(
+                (
+                    parsed_host.scheme or "http",
+                    parsed_host.netloc or parsed_host.path,
+                    "",
+                    "",
+                    "",
+                )
+            )
+
+        tmp_dir = tempfile.mkdtemp()
+        try:
+            # TODO: would be nice to also update the hyperlinks to these images, e.g. when you have <a href="{url}"><img src="{url}"></a>
+            for img_elt in xml_tools.find_all(top_elt, names=["img"]):
+                yield self.img_filters(client, img_elt, options, opt_host, tmp_dir)
+        finally:
+            os.rmdir(tmp_dir)  # XXX: tmp_dir should be empty, or something went wrong
+
+        # we now replace the content with filtered one
+        mb_data["content_xhtml"] = top_elt.toXml()
+
+    @defer.inlineCallbacks
+    def img_filters(self, client, img_elt, options, opt_host, tmp_dir):
+        """Filters handling images
+
+        url without host are fixed (if possible)
+        according to options, images are uploaded to XMPP server
+        @param img_elt(domish.Element): <img/> element to handle
+        @param options(dict): filters options
+        @param opt_host(unicode): normalised host given in options
+        @param tmp_dir(str): path to temp directory
+        """
+        try:
+            url = img_elt["src"]
+            if url[0] == "/":
+                if not opt_host:
+                    log.warning(
+                        "host was not specified, we can't deal with src without host ({url}) and have to ignore the following <img/>:\n{xml}".format(
+                            url=url, xml=img_elt.toXml()
+                        )
+                    )
+                    return
+                else:
+                    url = urllib.parse.urljoin(opt_host, url)
+            filename = url.rsplit("/", 1)[-1].strip()
+            if not filename:
+                raise KeyError
+        except (KeyError, IndexError):
+            log.warning("ignoring invalid img element: {}".format(img_elt.toXml()))
+            return
+
+        # we change the url for the normalized one
+        img_elt["src"] = url
+
+        if options.get(OPT_UPLOAD_IMAGES, False):
+            # upload is requested
+            try:
+                ignore_host = options[OPT_UPLOAD_IGNORE_HOST]
+            except KeyError:
+                pass
+            else:
+                # host is the ignored one, we skip
+                parsed_url = urllib.parse.urlsplit(url)
+                if ignore_host in parsed_url.hostname:
+                    log.info(
+                        "Don't upload image at {url} because of {opt} option".format(
+                            url=url, opt=OPT_UPLOAD_IGNORE_HOST
+                        )
+                    )
+                    return
+
+            # we download images and re-upload them via XMPP
+            tmp_file = os.path.join(tmp_dir, filename).encode("utf-8")
+            upload_options = {"ignore_tls_errors": options.get(OPT_IGNORE_TLS, False)}
+
+            try:
+                yield web_client.downloadPage(url.encode("utf-8"), tmp_file)
+                filename = filename.replace(
+                    "%", "_"
+                )  # FIXME: tmp workaround for a bug in prosody http upload
+                __, download_d = yield self._u.upload(
+                    client, tmp_file, filename, extra=upload_options
+                )
+                download_url = yield download_d
+            except Exception as e:
+                log.warning(
+                    "can't download image at {url}: {reason}".format(url=url, reason=e)
+                )
+            else:
+                img_elt["src"] = download_url
+
+            try:
+                os.unlink(tmp_file)
+            except OSError:
+                pass

libervia/backend/plugins/plugin_blog_import_dokuwiki.py

@@ -0,0 +1,414 @@
+#!/usr/bin/env python3
+
+
+# SàT plugin to import dokuwiki blogs
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+# Copyright (C) 2013-2016 Adrien Cossa (souliane@mailoo.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from libervia.backend.core.i18n import _, D_
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.log import getLogger
+
+log = getLogger(__name__)
+from libervia.backend.core import exceptions
+from libervia.backend.tools import xml_tools
+from twisted.internet import threads
+from collections import OrderedDict
+import calendar
+import urllib.request, urllib.parse, urllib.error
+import urllib.parse
+import tempfile
+import re
+import time
+import os.path
+
+try:
+    from dokuwiki import DokuWiki, DokuWikiError  # this is a new dependency
+except ImportError:
+    raise exceptions.MissingModule(
+        'Missing module dokuwiki, please install it with "pip install dokuwiki"'
+    )
+try:
+    from PIL import Image  # this is already needed by plugin XEP-0054
+except:
+    raise exceptions.MissingModule(
+        "Missing module pillow, please download/install it from https://python-pillow.github.io"
+    )
+
+PLUGIN_INFO = {
+    C.PI_NAME: "Dokuwiki import",
+    C.PI_IMPORT_NAME: "IMPORT_DOKUWIKI",
+    C.PI_TYPE: C.PLUG_TYPE_BLOG,
+    C.PI_DEPENDENCIES: ["BLOG_IMPORT"],
+    C.PI_MAIN: "DokuwikiImport",
+    C.PI_HANDLER: "no",
+    C.PI_DESCRIPTION: _("""Blog importer for Dokuwiki blog engine."""),
+}
+
+SHORT_DESC = D_("import posts from Dokuwiki blog engine")
+
+LONG_DESC = D_(
+    """This importer handle Dokuwiki blog engine.
+
+To use it, you need an admin access to a running Dokuwiki website
+(local or on the Internet). The importer retrieves the data using
+the XMLRPC Dokuwiki API.
+
+You can specify a namespace (that could be a namespace directory
+or a single post) or leave it empty to use the root namespace "/"
+and import all the posts.
+
+You can specify a new media repository to modify the internal
+media links and make them point to the URL of your choice, but
+note that the upload is not done automatically: a temporary
+directory will be created on your local drive and you will
+need to upload it yourself to your repository via SSH or FTP.
+
+Following options are recognized:
+
+location: DokuWiki site URL
+user: DokuWiki admin user
+passwd: DokuWiki admin password
+namespace: DokuWiki namespace to import (default: root namespace "/")
+media_repo: URL to the new remote media repository (default: none)
+limit: maximal number of posts to import (default: 100)
+
+Example of usage (with jp frontend):
+
+jp import dokuwiki -p dave --pwd xxxxxx --connect
+    http://127.0.1.1 -o user souliane -o passwd qwertz
+    -o namespace public:2015:10
+    -o media_repo http://media.diekulturvermittlung.at
+
+This retrieves the 100 last blog posts from http://127.0.1.1 that
+are inside the namespace "public:2015:10" using the Dokuwiki user
+"souliane", and it imports them to sat profile dave's microblog node.
+Internal Dokuwiki media that were hosted on http://127.0.1.1 are now
+pointing to http://media.diekulturvermittlung.at.
+"""
+)
+DEFAULT_MEDIA_REPO = ""
+DEFAULT_NAMESPACE = "/"
+DEFAULT_LIMIT = 100  # you might get a DBUS timeout (no reply) if it lasts too long
+
+
+class Importer(DokuWiki):
+    def __init__(
+        self, url, user, passwd, media_repo=DEFAULT_MEDIA_REPO, limit=DEFAULT_LIMIT
+    ):
+        """
+
+        @param url (unicode): DokuWiki site URL
+        @param user (unicode): DokuWiki admin user
+        @param passwd (unicode): DokuWiki admin password
+        @param media_repo (unicode): New remote media repository
+        """
+        DokuWiki.__init__(self, url, user, passwd)
+        self.url = url
+        self.media_repo = media_repo
+        self.temp_dir = tempfile.mkdtemp() if self.media_repo else None
+        self.limit = limit
+        self.posts_data = OrderedDict()
+
+    def get_post_id(self, post):
+        """Return a unique and constant post id
+
+        @param post(dict): parsed post data
+        @return (unicode): post unique item id
+        """
+        return str(post["id"])
+
+    def get_post_updated(self, post):
+        """Return the update date.
+
+        @param post(dict): parsed post data
+        @return (unicode): update date
+        """
+        return str(post["mtime"])
+
+    def get_post_published(self, post):
+        """Try to parse the date from the message ID, else use "mtime".
+
+        The date can be extracted if the message ID looks like one of:
+            - namespace:YYMMDD_short_title
+            - namespace:YYYYMMDD_short_title
+        @param post (dict):  parsed post data
+        @return (unicode): publication date
+        """
+        id_, default = str(post["id"]), str(post["mtime"])
+        try:
+            date = id_.split(":")[-1].split("_")[0]
+        except KeyError:
+            return default
+        try:
+            time_struct = time.strptime(date, "%y%m%d")
+        except ValueError:
+            try:
+                time_struct = time.strptime(date, "%Y%m%d")
+            except ValueError:
+                return default
+        return str(calendar.timegm(time_struct))
+
+    def process_post(self, post, profile_jid):
+        """Process a single page.
+
+        @param post (dict): parsed post data
+        @param profile_jid
+        """
+        # get main information
+        id_ = self.get_post_id(post)
+        updated = self.get_post_updated(post)
+        published = self.get_post_published(post)
+
+        # manage links
+        backlinks = self.pages.backlinks(id_)
+        for link in self.pages.links(id_):
+            if link["type"] != "extern":
+                assert link["type"] == "local"
+                page = link["page"]
+                backlinks.append(page[1:] if page.startswith(":") else page)
+
+        self.pages.get(id_)
+        content_xhtml = self.process_content(self.pages.html(id_), backlinks, profile_jid)
+
+        # XXX: title is already in content_xhtml and difficult to remove, so leave it
+        # title = content.split("\n")[0].strip(u"\ufeff= ")
+
+        # build the extra data dictionary
+        mb_data = {
+            "id": id_,
+            "published": published,
+            "updated": updated,
+            "author": profile_jid.user,
+            # "content": content,  # when passed, it is displayed in Libervia instead of content_xhtml
+            "content_xhtml": content_xhtml,
+            # "title": title,
+            "allow_comments": "true",
+        }
+
+        # find out if the message access is public or restricted
+        namespace = id_.split(":")[0]
+        if namespace and namespace.lower() not in ("public", "/"):
+            mb_data["group"] = namespace  # roster group must exist
+
+        self.posts_data[id_] = {"blog": mb_data, "comments": [[]]}
+
+    def process(self, client, namespace=DEFAULT_NAMESPACE):
+        """Process a namespace or a single page.
+
+        @param namespace (unicode): DokuWiki namespace (or page) to import
+        """
+        profile_jid = client.jid
+        log.info("Importing data from DokuWiki %s" % self.version)
+        try:
+            pages_list = self.pages.list(namespace)
+        except DokuWikiError:
+            log.warning(
+                'Could not list Dokuwiki pages: please turn the "display_errors" setting to "Off" in the php.ini of the webserver hosting DokuWiki.'
+            )
+            return
+
+        if not pages_list:  # namespace is actually a page?
+            names = namespace.split(":")
+            real_namespace = ":".join(names[0:-1])
+            pages_list = self.pages.list(real_namespace)
+            pages_list = [page for page in pages_list if page["id"] == namespace]
+            namespace = real_namespace
+
+        count = 0
+        for page in pages_list:
+            self.process_post(page, profile_jid)
+            count += 1
+            if count >= self.limit:
+                break
+
+        return (iter(self.posts_data.values()), len(self.posts_data))
+
+    def process_content(self, text, backlinks, profile_jid):
+        """Do text substitutions and file copy.
+
+        @param text (unicode): message content
+        @param backlinks (list[unicode]): list of backlinks
+        """
+        text = text.strip("\ufeff")  # this is at the beginning of the file (BOM)
+
+        for backlink in backlinks:
+            src = '/doku.php?id=%s"' % backlink
+            tgt = '/blog/%s/%s" target="#"' % (profile_jid.user, backlink)
+            text = text.replace(src, tgt)
+
+        subs = {}
+
+        link_pattern = r"""<(img|a)[^>]* (src|href)="([^"]+)"[^>]*>"""
+        for tag in re.finditer(link_pattern, text):
+            type_, attr, link = tag.group(1), tag.group(2), tag.group(3)
+            assert (type_ == "img" and attr == "src") or (type_ == "a" and attr == "href")
+            if re.match(r"^\w*://", link):  # absolute URL to link directly
+                continue
+            if self.media_repo:
+                self.move_media(link, subs)
+            elif link not in subs:
+                subs[link] = urllib.parse.urljoin(self.url, link)
+
+        for url, new_url in subs.items():
+            text = text.replace(url, new_url)
+        return text
+
+    def move_media(self, link, subs):
+        """Move a media from the DokuWiki host to the new repository.
+
+        This also updates the hyperlinks to internal media files.
+        @param link (unicode): media link
+        @param subs (dict): substitutions data
+        """
+        url = urllib.parse.urljoin(self.url, link)
+        user_media = re.match(r"(/lib/exe/\w+.php\?)(.*)", link)
+        thumb_width = None
+
+        if user_media:  # media that has been added by the user
+            params = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
+            try:
+                media = params["media"][0]
+            except KeyError:
+                log.warning("No media found in fetch URL: %s" % user_media.group(2))
+                return
+            if re.match(r"^\w*://", media):  # external URL to link directly
+                subs[link] = media
+                return
+            try:  # create thumbnail
+                thumb_width = params["w"][0]
+            except KeyError:
+                pass
+
+            filename = media.replace(":", "/")
+            # XXX: avoid "precondition failed" error (only keep the media parameter)
+            url = urllib.parse.urljoin(self.url, "/lib/exe/fetch.php?media=%s" % media)
+
+        elif link.startswith("/lib/plugins/"):
+            # other link added by a plugin or something else
+            filename = link[13:]
+        else:  # fake alert... there's no media (or we don't handle it yet)
+            return
+
+        filepath = os.path.join(self.temp_dir, filename)
+        self.download_media(url, filepath)
+
+        if thumb_width:
+            filename = os.path.join("thumbs", thumb_width, filename)
+            thumbnail = os.path.join(self.temp_dir, filename)
+            self.create_thumbnail(filepath, thumbnail, thumb_width)
+
+        new_url = os.path.join(self.media_repo, filename)
+        subs[link] = new_url
+
+    def download_media(self, source, dest):
+        """Copy media to localhost.
+
+        @param source (unicode): source url
+        @param dest (unicode): target path
+        """
+        dirname = os.path.dirname(dest)
+        if not os.path.exists(dest):
+            if not os.path.exists(dirname):
+                os.makedirs(dirname)
+            urllib.request.urlretrieve(source, dest)
+            log.debug("DokuWiki media file copied to %s" % dest)
+
+    def create_thumbnail(self, source, dest, width):
+        """Create a thumbnail.
+
+        @param source (unicode): source file path
+        @param dest (unicode): destination file path
+        @param width (unicode): thumbnail's width
+        """
+        thumb_dir = os.path.dirname(dest)
+        if not os.path.exists(thumb_dir):
+            os.makedirs(thumb_dir)
+        try:
+            im = Image.open(source)
+            im.thumbnail((width, int(width) * im.size[0] / im.size[1]))
+            im.save(dest)
+            log.debug("DokuWiki media thumbnail created: %s" % dest)
+        except IOError:
+            log.error("Cannot create DokuWiki media thumbnail %s" % dest)
+
+
+class DokuwikiImport(object):
+    def __init__(self, host):
+        log.info(_("plugin Dokuwiki import initialization"))
+        self.host = host
+        self._blog_import = host.plugins["BLOG_IMPORT"]
+        self._blog_import.register("dokuwiki", self.dk_import, SHORT_DESC, LONG_DESC)
+
+    def dk_import(self, client, location, options=None):
+        """import from DokuWiki to PubSub
+
+        @param location (unicode): DokuWiki site URL
+        @param options (dict, None): DokuWiki import parameters
+            - user (unicode): DokuWiki admin user
+            - passwd (unicode): DokuWiki admin password
+            - namespace (unicode): DokuWiki namespace to import
+            - media_repo (unicode): New remote media repository
+        """
+        options[self._blog_import.OPT_HOST] = location
+        try:
+            user = options["user"]
+        except KeyError:
+            raise exceptions.DataError('parameter "user" is required')
+        try:
+            passwd = options["passwd"]
+        except KeyError:
+            raise exceptions.DataError('parameter "passwd" is required')
+
+        opt_upload_images = options.get(self._blog_import.OPT_UPLOAD_IMAGES, None)
+        try:
+            media_repo = options["media_repo"]
+            if opt_upload_images:
+                options[
+                    self._blog_import.OPT_UPLOAD_IMAGES
+                ] = False  # force using --no-images-upload
+            info_msg = _(
+                "DokuWiki media files will be *downloaded* to {temp_dir} - to finish the import you have to upload them *manually* to {media_repo}"
+            )
+        except KeyError:
+            media_repo = DEFAULT_MEDIA_REPO
+            if opt_upload_images:
+                info_msg = _(
+                    "DokuWiki media files will be *uploaded* to the XMPP server. Hyperlinks to these media may not been updated though."
+                )
+            else:
+                info_msg = _(
+                    "DokuWiki media files will *stay* on {location} - some of them may be protected by DokuWiki ACL and will not be accessible."
+                )
+
+        try:
+            namespace = options["namespace"]
+        except KeyError:
+            namespace = DEFAULT_NAMESPACE
+        try:
+            limit = options["limit"]
+        except KeyError:
+            limit = DEFAULT_LIMIT
+
+        dk_importer = Importer(location, user, passwd, media_repo, limit)
+        info_msg = info_msg.format(
+            temp_dir=dk_importer.temp_dir, media_repo=media_repo, location=location
+        )
+        self.host.action_new(
+            {"xmlui": xml_tools.note(info_msg).toXml()}, profile=client.profile
+        )
+        d = threads.deferToThread(dk_importer.process, client, namespace)
+        return d

libervia/backend/plugins/plugin_blog_import_dotclear.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+
+
+# SàT plugin for import external blogs
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from libervia.backend.core.i18n import _, D_
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.log import getLogger
+
+log = getLogger(__name__)
+from libervia.backend.core import exceptions
+from libervia.backend.tools.common import data_format
+from twisted.internet import threads
+from collections import OrderedDict
+import itertools
+import time
+import cgi
+import os.path
+
+
+PLUGIN_INFO = {
+    C.PI_NAME: "Dotclear import",
+    C.PI_IMPORT_NAME: "IMPORT_DOTCLEAR",
+    C.PI_TYPE: C.PLUG_TYPE_BLOG,
+    C.PI_DEPENDENCIES: ["BLOG_IMPORT"],
+    C.PI_MAIN: "DotclearImport",
+    C.PI_HANDLER: "no",
+    C.PI_DESCRIPTION: _("""Blog importer for Dotclear blog engine."""),
+}
+
+SHORT_DESC = D_("import posts from Dotclear blog engine")
+
+LONG_DESC = D_(
+    """This importer handle Dotclear blog engine.
+
+To use it, you'll need to export your blog to a flat file.
+You must go in your admin interface and select Plugins/Maintenance then Backup.
+Export only one blog if you have many, i.e. select "Download database of current blog"
+Depending on your configuration, your may need to use import/Export plugin and export as a flat file.
+
+location: you must use the absolute path to your backup for the location parameter
+"""
+)
+POST_ID_PREFIX = "sat_dc_"
+KNOWN_DATA_TYPES = (
+    "link",
+    "setting",
+    "post",
+    "meta",
+    "media",
+    "post_media",
+    "comment",
+    "captcha",
+)
+ESCAPE_MAP = {"r": "\r", "n": "\n", '"': '"', "\\": "\\"}
+
+
+class DotclearParser(object):
+    # XXX: we have to parse all file to build data
+    #      this can be ressource intensive on huge blogs
+
+    def __init__(self):
+        self.posts_data = OrderedDict()
+        self.tags = {}
+
+    def get_post_id(self, post):
+        """Return a unique and constant post id
+
+        @param post(dict): parsed post data
+        @return (unicode): post unique item id
+        """
+        return "{}_{}_{}_{}:{}".format(
+            POST_ID_PREFIX,
+            post["blog_id"],
+            post["user_id"],
+            post["post_id"],
+            post["post_url"],
+        )
+
+    def get_comment_id(self, comment):
+        """Return a unique and constant comment id
+
+        @param comment(dict): parsed comment
+        @return (unicode): comment unique comment id
+        """
+        post_id = comment["post_id"]
+        parent_item_id = self.posts_data[post_id]["blog"]["id"]
+        return "{}_comment_{}".format(parent_item_id, comment["comment_id"])
+
+    def getTime(self, data, key):
+        """Parse time as given by dotclear, with timezone handling
+
+        @param data(dict): dotclear data (post or comment)
+        @param key(unicode): key to get (e.g. "post_creadt")
+        @return (float): Unix time
+        """
+        return time.mktime(time.strptime(data[key], "%Y-%m-%d %H:%M:%S"))
+
+    def read_fields(self, fields_data):
+        buf = []
+        idx = 0
+        while True:
+            if fields_data[idx] != '"':
+                raise exceptions.ParsingError
+            while True:
+                idx += 1
+                try:
+                    char = fields_data[idx]
+                except IndexError:
+                    raise exceptions.ParsingError("Data was expected")
+                if char == '"':
+                    # we have reached the end of this field,
+                    # we try to parse a new one
+                    yield "".join(buf)
+                    buf = []
+                    idx += 1
+                    try:
+                        separator = fields_data[idx]
+                    except IndexError:
+                        return
+                    if separator != ",":
+                        raise exceptions.ParsingError("Field separator was expeceted")
+                    idx += 1
+                    break  # we have a new field
+                elif char == "\\":
+                    idx += 1
+                    try:
+                        char = ESCAPE_MAP[fields_data[idx]]
+                    except IndexError:
+                        raise exceptions.ParsingError("Escaped char was expected")
+                    except KeyError:
+                        char = fields_data[idx]
+                        log.warning("Unknown key to escape: {}".format(char))
+                buf.append(char)
+
+    def parseFields(self, headers, data):
+        return dict(zip(headers, self.read_fields(data)))
+
+    def post_handler(self, headers, data, index):
+        post = self.parseFields(headers, data)
+        log.debug("({}) post found: {}".format(index, post["post_title"]))
+        mb_data = {
+            "id": self.get_post_id(post),
+            "published": self.getTime(post, "post_creadt"),
+            "updated": self.getTime(post, "post_upddt"),
+            "author": post["user_id"],  # there use info are not in the archive
+            # TODO: option to specify user info
+            "content_xhtml": "{}{}".format(
+                post["post_content_xhtml"], post["post_excerpt_xhtml"]
+            ),
+            "title": post["post_title"],
+            "allow_comments": C.bool_const(bool(int(post["post_open_comment"]))),
+        }
+        self.posts_data[post["post_id"]] = {
+            "blog": mb_data,
+            "comments": [[]],
+            "url": "/post/{}".format(post["post_url"]),
+        }
+
+    def meta_handler(self, headers, data, index):
+        meta = self.parseFields(headers, data)
+        if meta["meta_type"] == "tag":
+            tags = self.tags.setdefault(meta["post_id"], set())
+            tags.add(meta["meta_id"])
+
+    def meta_finished_handler(self):
+        for post_id, tags in self.tags.items():
+            data_format.iter2dict("tag", tags, self.posts_data[post_id]["blog"])
+        del self.tags
+
+    def comment_handler(self, headers, data, index):
+        comment = self.parseFields(headers, data)
+        if comment["comment_site"]:
+            # we don't use atom:uri because it's used for jid in XMPP
+            content = '{}\n<hr>\n<a href="{}">author website</a>'.format(
+                comment["comment_content"],
+                cgi.escape(comment["comment_site"]).replace('"', "%22"),
+            )
+        else:
+            content = comment["comment_content"]
+        mb_data = {
+            "id": self.get_comment_id(comment),
+            "published": self.getTime(comment, "comment_dt"),
+            "updated": self.getTime(comment, "comment_upddt"),
+            "author": comment["comment_author"],
+            # we don't keep email addresses to avoid the author to be spammed
+            # (they would be available publicly else)
+            # 'author_email': comment['comment_email'],
+            "content_xhtml": content,
+        }
+        self.posts_data[comment["post_id"]]["comments"][0].append(
+            {"blog": mb_data, "comments": [[]]}
+        )
+
+    def parse(self, db_path):
+        with open(db_path) as f:
+            signature = f.readline()
+            try:
+                version = signature.split("|")[1]
+            except IndexError:
+                version = None
+            log.debug("Dotclear version: {}".format(version))
+            data_type = None
+            data_headers = None
+            index = None
+            while True:
+                buf = f.readline()
+                if not buf:
+                    break
+                if buf.startswith("["):
+                    header = buf.split(" ", 1)
+                    data_type = header[0][1:]
+                    if data_type not in KNOWN_DATA_TYPES:
+                        log.warning("unkown data type: {}".format(data_type))
+                    index = 0
+                    try:
+                        data_headers = header[1].split(",")
+                        # we need to remove the ']' from the last header
+                        last_header = data_headers[-1]
+                        data_headers[-1] = last_header[: last_header.rfind("]")]
+                    except IndexError:
+                        log.warning("Can't read data)")
+                else:
+                    if data_type is None:
+                        continue
+                    buf = buf.strip()
+                    if not buf and data_type in KNOWN_DATA_TYPES:
+                        try:
+                            finished_handler = getattr(
+                                self, "{}FinishedHandler".format(data_type)
+                            )
+                        except AttributeError:
+                            pass
+                        else:
+                            finished_handler()
+                        log.debug("{} data finished".format(data_type))
+                        data_type = None
+                        continue
+                    assert data_type
+                    try:
+                        fields_handler = getattr(self, "{}Handler".format(data_type))
+                    except AttributeError:
+                        pass
+                    else:
+                        fields_handler(data_headers, buf, index)
+                    index += 1
+        return (iter(self.posts_data.values()), len(self.posts_data))
+
+
+class DotclearImport(object):
+    def __init__(self, host):
+        log.info(_("plugin Dotclear import initialization"))
+        self.host = host
+        host.plugins["BLOG_IMPORT"].register(
+            "dotclear", self.dc_import, SHORT_DESC, LONG_DESC
+        )
+
+    def dc_import(self, client, location, options=None):
+        if not os.path.isabs(location):
+            raise exceptions.DataError(
+                "An absolute path to backup data need to be given as location"
+            )
+        dc_parser = DotclearParser()
+        d = threads.deferToThread(dc_parser.parse, location)
+        return d

libervia/backend/plugins/plugin_comp_ap_gateway/ad_hoc.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+# Libervia ActivityPub Gateway
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from twisted.words.protocols.jabber import jid
+from twisted.words.xish import domish
+from wokkel import data_form
+
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.core_types import SatXMPPEntity
+from libervia.backend.core.i18n import _
+from libervia.backend.core.log import getLogger
+
+
+log = getLogger(__name__)
+NS_XMPP_JID_NODE_2_AP = "https://libervia.org/ap_gateway/xmpp_jid_node_2_ap_actor"
+
+class APAdHocService:
+    """Ad-Hoc commands for AP Gateway"""
+
+    def __init__(self, apg):
+        self.host = apg.host
+        self.apg = apg
+        self._c = self.host.plugins["XEP-0050"]
+
+    def init(self, client: SatXMPPEntity) -> None:
+        self._c.add_ad_hoc_command(
+            client,
+            self.xmpp_jid_node_2_ap_actor,
+            "Convert XMPP JID/Node to AP actor",
+            node=NS_XMPP_JID_NODE_2_AP,
+            allowed_magics=C.ENTITY_ALL,
+        )
+
+    async def xmpp_jid_node_2_ap_actor(
+        self,
+        client: SatXMPPEntity,
+        command_elt: domish.Element,
+        session_data: dict,
+        action: str,
+        node: str
+    ):
+        try:
+            x_elt = next(command_elt.elements(data_form.NS_X_DATA, "x"))
+            command_form = data_form.Form.fromElement(x_elt)
+        except StopIteration:
+            command_form = None
+        if command_form is None or len(command_form.fields) == 0:
+            # root request
+            status = self._c.STATUS.EXECUTING
+            form = data_form.Form(
+                "form", title="XMPP JID/node to AP actor conversion",
+                formNamespace=NS_XMPP_JID_NODE_2_AP
+            )
+
+            field = data_form.Field(
+                "text-single", "jid", required=True
+            )
+            form.addField(field)
+
+            field = data_form.Field(
+                "text-single", "node", required=False
+            )
+            form.addField(field)
+
+            payload = form.toElement()
+            return payload, status, None, None
+        else:
+            xmpp_jid = jid.JID(command_form["jid"])
+            xmpp_node = command_form.get("node")
+            actor = await self.apg.get_ap_account_from_jid_and_node(xmpp_jid, xmpp_node)
+            note = (self._c.NOTE.INFO, actor)
+            status = self._c.STATUS.COMPLETED
+            payload = None
+            return (payload, status, None, note)

libervia/backend/plugins/plugin_comp_ap_gateway/constants.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+# Libervia ActivityPub Gateway
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+IMPORT_NAME = "ap-gateway"
+CONF_SECTION = f"component {IMPORT_NAME}"
+CONTENT_TYPE_AP = "application/activity+json; charset=utf-8"
+TYPE_ACTOR = "actor"
+TYPE_INBOX = "inbox"
+TYPE_SHARED_INBOX = "shared_inbox"
+TYPE_OUTBOX = "outbox"
+TYPE_FOLLOWERS = "followers"
+TYPE_FOLLOWING = "following"
+TYPE_ITEM = "item"
+TYPE_TOMBSTONE = "Tombstone"
+TYPE_MENTION = "Mention"
+TYPE_LIKE = "Like"
+TYPE_REACTION = "EmojiReact"
+TYPE_EVENT = "Event"
+TYPE_JOIN = "Join"
+TYPE_LEAVE = "Leave"
+MEDIA_TYPE_AP = "application/activity+json"
+NS_AP = "https://www.w3.org/ns/activitystreams"
+NS_AP_PUBLIC = f"{NS_AP}#Public"
+# 3 values can be used, see https://www.w3.org/TR/activitypub/#public-addressing
+PUBLIC_TUPLE = (NS_AP_PUBLIC, "as:Public", "Public")
+AP_REQUEST_TYPES = {
+    "GET": {TYPE_ACTOR, TYPE_OUTBOX, TYPE_FOLLOWERS, TYPE_FOLLOWING},
+    "POST": {"inbox"},
+}
+AP_REQUEST_TYPES["HEAD"] = AP_REQUEST_TYPES["GET"]
+# headers to check for signature
+SIGN_HEADERS = {
+    # headers needed for all HTTP methods
+    None: [
+        # tuples are equivalent headers/pseudo headers, one of them must be present
+        ("date", "(created)"),
+        ("digest", "(request-target)"),
+    ],
+    b"GET": ["host"],
+    b"POST": ["digest"]
+}
+PAGE_SIZE = 10
+HS2019 = "hs2019"
+# delay after which a signed request is not accepted anymore
+SIGN_EXP = 12*60*60  # 12 hours (same value as for Mastodon)
+
+LRU_MAX_SIZE = 200
+ACTIVITY_TYPES = (
+    "Accept", "Add", "Announce", "Arrive", "Block", "Create", "Delete", "Dislike", "Flag",
+    "Follow", "Ignore", "Invite", "Join", "Leave", "Like", "Listen", "Move", "Offer",
+    "Question", "Reject", "Read", "Remove", "TentativeReject", "TentativeAccept",
+    "Travel", "Undo", "Update", "View",
+    # non-standard activities
+    "EmojiReact"
+)
+ACTIVITY_TYPES_LOWER = [a.lower() for a in ACTIVITY_TYPES]
+ACTIVITY_OBJECT_MANDATORY = (
+    "Create", "Update", "Delete", "Follow", "Add", "Remove", "Like", "Block", "Undo"
+)
+ACTIVITY_TARGET_MANDATORY = ("Add", "Remove")
+# activities which can be used with Shared Inbox (i.e. with no account specified)
+# must be lowercase
+ACTIVIY_NO_ACCOUNT_ALLOWED = (
+    "create", "update", "delete", "announce", "undo", "like", "emojireact", "join",
+    "leave"
+)
+# maximum number of parents to retrieve when comments_max_depth option is set
+COMMENTS_MAX_PARENTS = 100
+# maximum size of avatar, in bytes
+MAX_AVATAR_SIZE = 1024 * 1024 * 5
+
+# storage prefixes
+ST_AVATAR = "[avatar]"
+ST_AP_CACHE = "[AP_item_cache]"

libervia/backend/plugins/plugin_comp_ap_gateway/events.py

@@ -0,0 +1,407 @@
+#!/usr/bin/env python3
+
+# Libervia ActivityPub Gateway
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import Tuple
+
+import mimetypes
+import html
+
+import shortuuid
+from twisted.words.xish import domish
+from twisted.words.protocols.jabber import jid
+
+from libervia.backend.core.i18n import _
+from libervia.backend.core.log import getLogger
+from libervia.backend.core import exceptions
+from libervia.backend.tools.common import date_utils, uri
+
+from .constants import NS_AP_PUBLIC, TYPE_ACTOR, TYPE_EVENT, TYPE_ITEM
+
+
+log = getLogger(__name__)
+
+# direct copy of what Mobilizon uses
+AP_EVENTS_CONTEXT = {
+    "@language": "und",
+    "Hashtag": "as:Hashtag",
+    "PostalAddress": "sc:PostalAddress",
+    "PropertyValue": "sc:PropertyValue",
+    "address": {"@id": "sc:address", "@type": "sc:PostalAddress"},
+    "addressCountry": "sc:addressCountry",
+    "addressLocality": "sc:addressLocality",
+    "addressRegion": "sc:addressRegion",
+    "anonymousParticipationEnabled": {"@id": "mz:anonymousParticipationEnabled",
+                                      "@type": "sc:Boolean"},
+    "category": "sc:category",
+    "commentsEnabled": {"@id": "pt:commentsEnabled",
+                        "@type": "sc:Boolean"},
+    "discoverable": "toot:discoverable",
+    "discussions": {"@id": "mz:discussions", "@type": "@id"},
+    "events": {"@id": "mz:events", "@type": "@id"},
+    "ical": "http://www.w3.org/2002/12/cal/ical#",
+    "inLanguage": "sc:inLanguage",
+    "isOnline": {"@id": "mz:isOnline", "@type": "sc:Boolean"},
+    "joinMode": {"@id": "mz:joinMode", "@type": "mz:joinModeType"},
+    "joinModeType": {"@id": "mz:joinModeType",
+                     "@type": "rdfs:Class"},
+    "location": {"@id": "sc:location", "@type": "sc:Place"},
+    "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+    "maximumAttendeeCapacity": "sc:maximumAttendeeCapacity",
+    "memberCount": {"@id": "mz:memberCount", "@type": "sc:Integer"},
+    "members": {"@id": "mz:members", "@type": "@id"},
+    "mz": "https://joinmobilizon.org/ns#",
+    "openness": {"@id": "mz:openness", "@type": "@id"},
+    "participantCount": {"@id": "mz:participantCount",
+                         "@type": "sc:Integer"},
+    "participationMessage": {"@id": "mz:participationMessage",
+                             "@type": "sc:Text"},
+    "postalCode": "sc:postalCode",
+    "posts": {"@id": "mz:posts", "@type": "@id"},
+    "propertyID": "sc:propertyID",
+    "pt": "https://joinpeertube.org/ns#",
+    "remainingAttendeeCapacity": "sc:remainingAttendeeCapacity",
+    "repliesModerationOption": {"@id": "mz:repliesModerationOption",
+                                "@type": "mz:repliesModerationOptionType"},
+    "repliesModerationOptionType": {"@id": "mz:repliesModerationOptionType",
+                                    "@type": "rdfs:Class"},
+    "resources": {"@id": "mz:resources", "@type": "@id"},
+    "sc": "http://schema.org#",
+    "streetAddress": "sc:streetAddress",
+    "timezone": {"@id": "mz:timezone", "@type": "sc:Text"},
+    "todos": {"@id": "mz:todos", "@type": "@id"},
+    "toot": "http://joinmastodon.org/ns#",
+    "uuid": "sc:identifier",
+    "value": "sc:value"
+}
+
+
+class APEvents:
+    """XMPP Events <=> AP Events conversion"""
+
+    def __init__(self, apg):
+        self.host = apg.host
+        self.apg = apg
+        self._events = self.host.plugins["XEP-0471"]
+
+    async def event_data_2_ap_item(
+        self, event_data: dict, author_jid: jid.JID, is_new: bool=True
+    ) -> dict:
+        """Convert event data to AP activity
+
+        @param event_data: event data as used in [plugin_exp_events]
+        @param author_jid: jid of the published of the event
+        @param is_new: if True, the item is a new one (no instance has been found in
+            cache).
+            If True, a "Create" activity will be generated, otherwise an "Update" one will
+            be
+        @return: AP activity wrapping an Event object
+        """
+        if not event_data.get("id"):
+            event_data["id"] = shortuuid.uuid()
+        ap_account = await self.apg.get_ap_account_from_jid_and_node(
+            author_jid,
+            self._events.namespace
+        )
+        url_actor = self.apg.build_apurl(TYPE_ACTOR, ap_account)
+        url_item = self.apg.build_apurl(TYPE_ITEM, ap_account, event_data["id"])
+        ap_object = {
+            "actor": url_actor,
+            "attributedTo": url_actor,
+            "to": [NS_AP_PUBLIC],
+            "id": url_item,
+            "type": TYPE_EVENT,
+            "name": next(iter(event_data["name"].values())),
+            "startTime": date_utils.date_fmt(event_data["start"], "iso"),
+            "endTime": date_utils.date_fmt(event_data["end"], "iso"),
+            "url": url_item,
+        }
+
+        attachment = ap_object["attachment"] = []
+
+        # FIXME: we only handle URL head-picture for now
+        # TODO: handle jingle and use file metadata
+        try:
+            head_picture_url = event_data["head-picture"]["sources"][0]["url"]
+        except (KeyError, IndexError, TypeError):
+            pass
+        else:
+            media_type = mimetypes.guess_type(head_picture_url, False)[0] or "image/jpeg"
+            attachment.append({
+                "name": "Banner",
+                "type": "Document",
+                "mediaType": media_type,
+                "url": head_picture_url,
+            })
+
+        descriptions = event_data.get("descriptions")
+        if descriptions:
+            for description in descriptions:
+                content = description["description"]
+                if description["type"] == "xhtml":
+                    break
+            else:
+                content = f"<p>{html.escape(content)}</p>"  # type: ignore
+            ap_object["content"] = content
+
+        categories = event_data.get("categories")
+        if categories:
+            tag = ap_object["tag"] = []
+            for category in categories:
+                tag.append({
+                    "name": f"#{category['term']}",
+                    "type": "Hashtag",
+                })
+
+        locations = event_data.get("locations")
+        if locations:
+            ap_loc = ap_object["location"] = {}
+            # we only use the first found location
+            location = locations[0]
+            for source, dest in (
+                ("description", "name"),
+                ("lat", "latitude"),
+                ("lon", "longitude"),
+            ):
+                value = location.get(source)
+                if value is not None:
+                    ap_loc[dest] = value
+            for source, dest in (
+                ("country", "addressCountry"),
+                ("locality", "addressLocality"),
+                ("region", "addressRegion"),
+                ("postalcode", "postalCode"),
+                ("street", "streetAddress"),
+            ):
+                value = location.get(source)
+                if value is not None:
+                    ap_loc.setdefault("address", {})[dest] = value
+
+        if event_data.get("comments"):
+            ap_object["commentsEnabled"] = True
+
+        extra = event_data.get("extra")
+
+        if extra:
+            status = extra.get("status")
+            if status:
+                ap_object["ical:status"] = status.upper()
+
+            website = extra.get("website")
+            if website:
+                attachment.append({
+                    "href": website,
+                    "mediaType": "text/html",
+                    "name": "Website",
+                    "type": "Link"
+                })
+
+            accessibility = extra.get("accessibility")
+            if accessibility:
+                wheelchair = accessibility.get("wheelchair")
+                if wheelchair:
+                    if wheelchair == "full":
+                        ap_wc_value = "fully"
+                    elif wheelchair == "partial":
+                        ap_wc_value = "partially"
+                    elif wheelchair == "no":
+                        ap_wc_value = "no"
+                    else:
+                        log.error(f"unexpected wheelchair value: {wheelchair}")
+                        ap_wc_value = None
+                    if ap_wc_value is not None:
+                        attachment.append({
+                            "propertyID": "mz:accessibility:wheelchairAccessible",
+                            "type": "PropertyValue",
+                            "value": ap_wc_value
+                        })
+
+        activity = self.apg.create_activity(
+            "Create" if is_new else "Update", url_actor, ap_object, activity_id=url_item
+        )
+        activity["@context"].append(AP_EVENTS_CONTEXT)
+        return activity
+
+    async def ap_item_2_event_data(self, ap_item: dict) -> dict:
+        """Convert AP activity or object to event data
+
+        @param ap_item: ActivityPub item to convert
+            Can be either an activity of an object
+        @return: AP Item's Object and event data
+        @raise exceptions.DataError: something is invalid in the AP item
+        """
+        is_activity = self.apg.is_activity(ap_item)
+        if is_activity:
+            ap_object = await self.apg.ap_get_object(ap_item, "object")
+            if not ap_object:
+                log.warning(f'No "object" found in AP item {ap_item!r}')
+                raise exceptions.DataError
+        else:
+            ap_object = ap_item
+
+        # id
+        if "_repeated" in ap_item:
+            # if the event is repeated, we use the original one ID
+            repeated_uri = ap_item["_repeated"]["uri"]
+            parsed_uri = uri.parse_xmpp_uri(repeated_uri)
+            object_id = parsed_uri["item"]
+        else:
+            object_id = ap_object.get("id")
+            if not object_id:
+                raise exceptions.DataError('"id" is missing in AP object')
+
+        if ap_item["type"] != TYPE_EVENT:
+            raise exceptions.DataError("AP Object is not an event")
+
+        # author
+        actor = await self.apg.ap_get_sender_actor(ap_object)
+
+        account = await self.apg.get_ap_account_from_id(actor)
+        author_jid = self.apg.get_local_jid_from_account(account).full()
+
+        # name, start, end
+        event_data = {
+            "id": object_id,
+            "name": {"": ap_object.get("name") or "unnamed"},
+            "start": date_utils.date_parse(ap_object["startTime"]),
+            "end": date_utils.date_parse(ap_object["endTime"]),
+        }
+
+        # attachments/extra
+        event_data["extra"] = extra = {}
+        attachments = ap_object.get("attachment") or []
+        for attachment in attachments:
+            name = attachment.get("name")
+            if name == "Banner":
+                try:
+                    url = attachment["url"]
+                except KeyError:
+                    log.warning(f"invalid attachment: {attachment}")
+                    continue
+                event_data["head-picture"] = {"sources": [{"url": url}]}
+            elif name == "Website":
+                try:
+                    url = attachment["href"]
+                except KeyError:
+                    log.warning(f"invalid attachment: {attachment}")
+                    continue
+                extra["website"] = url
+            else:
+                log.debug(f"unmanaged attachment: {attachment}")
+
+        # description
+        content = ap_object.get("content")
+        if content:
+            event_data["descriptions"] = [{
+                "type": "xhtml",
+                "description": content
+            }]
+
+        # categories
+        tags = ap_object.get("tag")
+        if tags:
+            categories = event_data["categories"] = []
+            for tag in tags:
+                if tag.get("type") == "Hashtag":
+                    try:
+                        term = tag["name"][1:]
+                    except KeyError:
+                        log.warning(f"invalid tag: {tag}")
+                        continue
+                    categories.append({"term": term})
+
+        #location
+        ap_location = ap_object.get("location")
+        if ap_location:
+            location = {}
+            for source, dest in (
+                ("name", "description"),
+                ("latitude", "lat"),
+                ("longitude", "lon"),
+            ):
+                value = ap_location.get(source)
+                if value is not None:
+                    location[dest] = value
+            address = ap_location.get("address")
+            if address:
+                for source, dest in (
+                    ("addressCountry", "country"),
+                    ("addressLocality", "locality"),
+                    ("addressRegion", "region"),
+                    ("postalCode", "postalcode"),
+                    ("streetAddress", "street"),
+                ):
+                    value = address.get(source)
+                    if value is not None:
+                        location[dest] = value
+            if location:
+                event_data["locations"] = [location]
+
+        # rsvp
+        # So far Mobilizon seems to only handle participate/don't participate, thus we use
+        # a simple "yes"/"no" form.
+        rsvp_data = {"fields": []}
+        event_data["rsvp"] = [rsvp_data]
+        rsvp_data["fields"].append({
+            "type": "list-single",
+            "name": "attending",
+            "label": "Attending",
+            "options": [
+                {"label": "yes", "value": "yes"},
+                {"label": "no", "value": "no"}
+            ],
+            "required": True
+        })
+
+        # comments
+
+        if ap_object.get("commentsEnabled"):
+            __, comments_node = await self.apg.get_comments_nodes(object_id, None)
+            event_data["comments"] = {
+                "service": author_jid,
+                "node": comments_node,
+            }
+
+        # extra
+        # part of extra come from "attachment" above
+
+        status = ap_object.get("ical:status")
+        if status is None:
+            pass
+        elif status in ("CONFIRMED", "CANCELLED", "TENTATIVE"):
+            extra["status"] = status.lower()
+        else:
+            log.warning(f"unknown event status: {status}")
+
+        return event_data
+
+    async def ap_item_2_event_data_and_elt(
+        self,
+        ap_item: dict
+    ) -> Tuple[dict, domish.Element]:
+        """Convert AP item to parsed event data and corresponding item element"""
+        event_data = await self.ap_item_2_event_data(ap_item)
+        event_elt = self._events.event_data_2_event_elt(event_data)
+        item_elt = domish.Element((None, "item"))
+        item_elt["id"] = event_data["id"]
+        item_elt.addChild(event_elt)
+        return event_data, item_elt
+
+    async def ap_item_2_event_elt(self, ap_item: dict) -> domish.Element:
+        """Convert AP item to XMPP item element"""
+        __, item_elt = await self.ap_item_2_event_data_and_elt(ap_item)
+        return item_elt

libervia/backend/plugins/plugin_comp_ap_gateway/pubsub_service.py

@@ -0,0 +1,570 @@
+#!/usr/bin/env python3
+
+# Libervia ActivityPub Gateway
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import Optional, Tuple, List, Dict, Any, Union
+from urllib.parse import urlparse
+from pathlib import Path
+from base64 import b64encode
+import tempfile
+
+from twisted.internet import defer, threads
+from twisted.words.protocols.jabber import jid, error
+from twisted.words.xish import domish
+from wokkel import rsm, pubsub, disco
+
+from libervia.backend.core.i18n import _
+from libervia.backend.core import exceptions
+from libervia.backend.core.core_types import SatXMPPEntity
+from libervia.backend.core.log import getLogger
+from libervia.backend.core.constants import Const as C
+from libervia.backend.tools import image
+from libervia.backend.tools.utils import ensure_deferred
+from libervia.backend.tools.web import download_file
+from libervia.backend.memory.sqla_mapping import PubsubSub, SubscriptionState
+
+from .constants import (
+    TYPE_ACTOR,
+    ST_AVATAR,
+    MAX_AVATAR_SIZE
+)
+
+
+log = getLogger(__name__)
+
+# all nodes have the same config
+NODE_CONFIG = [
+    {"var": "pubsub#persist_items", "type": "boolean", "value": True},
+    {"var": "pubsub#max_items", "value": "max"},
+    {"var": "pubsub#access_model", "type": "list-single", "value": "open"},
+    {"var": "pubsub#publish_model", "type": "list-single", "value": "open"},
+
+]
+
+NODE_CONFIG_VALUES = {c["var"]: c["value"] for c in NODE_CONFIG}
+NODE_OPTIONS = {c["var"]: {} for c in NODE_CONFIG}
+for c in NODE_CONFIG:
+    NODE_OPTIONS[c["var"]].update({k:v for k,v in c.items() if k not in ("var", "value")})
+
+
+class APPubsubService(rsm.PubSubService):
+    """Pubsub service for XMPP requests"""
+
+    def __init__(self, apg):
+        super(APPubsubService, self).__init__()
+        self.host = apg.host
+        self.apg = apg
+        self.discoIdentity = {
+            "category": "pubsub",
+            "type": "service",
+            "name": "Libervia ActivityPub Gateway",
+        }
+
+    async def get_ap_actor_ids_and_inbox(
+        self,
+        requestor: jid.JID,
+        recipient: jid.JID,
+    ) -> Tuple[str, str, str]:
+        """Get AP actor IDs from requestor and destinee JIDs
+
+        @param requestor: XMPP entity doing a request to an AP actor via the gateway
+        @param recipient: JID mapping an AP actor via the gateway
+        @return: requestor actor ID, recipient actor ID and recipient inbox
+        @raise error.StanzaError: "item-not-found" is raised if not user part is specified
+            in requestor
+        """
+        if not recipient.user:
+            raise error.StanzaError(
+                "item-not-found",
+                text="No user part specified"
+            )
+        requestor_actor_id = self.apg.build_apurl(TYPE_ACTOR, requestor.userhost())
+        recipient_account = self.apg._e.unescape(recipient.user)
+        recipient_actor_id = await self.apg.get_ap_actor_id_from_account(recipient_account)
+        inbox = await self.apg.get_ap_inbox_from_id(recipient_actor_id, use_shared=False)
+        return requestor_actor_id, recipient_actor_id, inbox
+
+
+    @ensure_deferred
+    async def publish(self, requestor, service, nodeIdentifier, items):
+        if self.apg.local_only and not self.apg.is_local(requestor):
+            raise error.StanzaError(
+                "forbidden",
+                "Only local users can publish on this gateway."
+            )
+        if not service.user:
+            raise error.StanzaError(
+                "bad-request",
+                "You must specify an ActivityPub actor account in JID user part."
+            )
+        ap_account = self.apg._e.unescape(service.user)
+        if ap_account.count("@") != 1:
+            raise error.StanzaError(
+                "bad-request",
+                f"{ap_account!r} is not a valid ActivityPub actor account."
+            )
+
+        client = self.apg.client.get_virtual_client(requestor)
+        if self.apg._pa.is_attachment_node(nodeIdentifier):
+            await self.apg.convert_and_post_attachments(
+                client, ap_account, service, nodeIdentifier, items, publisher=requestor
+            )
+        else:
+            await self.apg.convert_and_post_items(
+                client, ap_account, service, nodeIdentifier, items
+            )
+            cached_node = await self.host.memory.storage.get_pubsub_node(
+                client, service, nodeIdentifier, with_subscriptions=True, create=True
+            )
+            await self.host.memory.storage.cache_pubsub_items(
+                client,
+                cached_node,
+                items
+            )
+            for subscription in cached_node.subscriptions:
+                if subscription.state != SubscriptionState.SUBSCRIBED:
+                    continue
+                self.notifyPublish(
+                    service,
+                    nodeIdentifier,
+                    [(subscription.subscriber, None, items)]
+                )
+
+    async def ap_following_2_elt(self, ap_item: dict) -> domish.Element:
+        """Convert actor ID from following collection to XMPP item"""
+        actor_id = ap_item["id"]
+        actor_jid = await self.apg.get_jid_from_id(actor_id)
+        subscription_elt = self.apg._pps.build_subscription_elt(
+            self.apg._m.namespace, actor_jid
+        )
+        item_elt = pubsub.Item(id=actor_id, payload=subscription_elt)
+        return item_elt
+
+    async def ap_follower_2_elt(self, ap_item: dict) -> domish.Element:
+        """Convert actor ID from followers collection to XMPP item"""
+        actor_id = ap_item["id"]
+        actor_jid = await self.apg.get_jid_from_id(actor_id)
+        subscriber_elt = self.apg._pps.build_subscriber_elt(actor_jid)
+        item_elt = pubsub.Item(id=actor_id, payload=subscriber_elt)
+        return item_elt
+
+    async def generate_v_card(self, ap_account: str) -> domish.Element:
+        """Generate vCard4 (XEP-0292) item element from ap_account's metadata"""
+        actor_data = await self.apg.get_ap_actor_data_from_account(ap_account)
+        identity_data = {}
+
+        summary = actor_data.get("summary")
+        # summary is HTML, we have to convert it to text
+        if summary:
+            identity_data["description"] = await self.apg._t.convert(
+                summary,
+                self.apg._t.SYNTAX_XHTML,
+                self.apg._t.SYNTAX_TEXT,
+                False,
+            )
+
+        for field in ("name", "preferredUsername"):
+            value = actor_data.get(field)
+            if value:
+                identity_data.setdefault("nicknames", []).append(value)
+        vcard_elt = self.apg._v.dict_2_v_card(identity_data)
+        item_elt = domish.Element((pubsub.NS_PUBSUB, "item"))
+        item_elt.addChild(vcard_elt)
+        item_elt["id"] = self.apg._p.ID_SINGLETON
+        return item_elt
+
+    async def get_avatar_data(
+        self,
+        client: SatXMPPEntity,
+        ap_account: str
+    ) -> Dict[str, Any]:
+        """Retrieve actor's avatar if any, cache it and file actor_data
+
+        ``cache_uid``, `path``` and ``media_type`` keys are always files
+        ``base64`` key is only filled if the file was not already in cache
+        """
+        actor_data = await self.apg.get_ap_actor_data_from_account(ap_account)
+
+        for icon in await self.apg.ap_get_list(actor_data, "icon"):
+            url = icon.get("url")
+            if icon["type"] != "Image" or not url:
+                continue
+            parsed_url = urlparse(url)
+            if not parsed_url.scheme in ("http", "https"):
+                log.warning(f"unexpected URL scheme: {url!r}")
+                continue
+            filename = Path(parsed_url.path).name
+            if not filename:
+                log.warning(f"ignoring URL with invald path: {url!r}")
+                continue
+            break
+        else:
+            raise error.StanzaError("item-not-found")
+
+        key = f"{ST_AVATAR}{url}"
+        cache_uid = await client._ap_storage.get(key)
+
+        if cache_uid is None:
+            cache = None
+        else:
+            cache = self.apg.host.common_cache.get_metadata(cache_uid)
+
+        if cache is None:
+            with tempfile.TemporaryDirectory() as dir_name:
+                dest_path = Path(dir_name, filename)
+                await download_file(url, dest_path, max_size=MAX_AVATAR_SIZE)
+                avatar_data = {
+                    "path": dest_path,
+                    "filename": filename,
+                    'media_type': image.guess_type(dest_path),
+                }
+
+                await self.apg._i.cache_avatar(
+                    self.apg.IMPORT_NAME,
+                    avatar_data
+                )
+        else:
+            avatar_data = {
+            "cache_uid": cache["uid"],
+            "path": cache["path"],
+            "media_type": cache["mime_type"]
+        }
+
+        return avatar_data
+
+    async def generate_avatar_metadata(
+        self,
+        client: SatXMPPEntity,
+        ap_account: str
+    ) -> domish.Element:
+        """Generate the metadata element for user avatar
+
+        @raise StanzaError("item-not-found"): no avatar is present in actor data (in
+            ``icon`` field)
+        """
+        avatar_data = await self.get_avatar_data(client, ap_account)
+        return self.apg._a.build_item_metadata_elt(avatar_data)
+
+    def _blocking_b_6_4_encode_avatar(self, avatar_data: Dict[str, Any]) -> None:
+        with avatar_data["path"].open("rb") as f:
+            avatar_data["base64"] = b64encode(f.read()).decode()
+
+    async def generate_avatar_data(
+        self,
+        client: SatXMPPEntity,
+        ap_account: str,
+        itemIdentifiers: Optional[List[str]],
+    ) -> domish.Element:
+        """Generate the data element for user avatar
+
+        @raise StanzaError("item-not-found"): no avatar cached with requested ID
+        """
+        if not itemIdentifiers:
+            avatar_data = await self.get_avatar_data(client, ap_account)
+            if "base64" not in avatar_data:
+                await threads.deferToThread(self._blocking_b_6_4_encode_avatar, avatar_data)
+        else:
+            if len(itemIdentifiers) > 1:
+                # only a single item ID is supported
+                raise error.StanzaError("item-not-found")
+            item_id = itemIdentifiers[0]
+            # just to be sure that that we don't have an empty string
+            assert item_id
+            cache_data = self.apg.host.common_cache.get_metadata(item_id)
+            if cache_data is None:
+                raise error.StanzaError("item-not-found")
+            avatar_data = {
+                "cache_uid": item_id,
+                "path": cache_data["path"]
+            }
+            await threads.deferToThread(self._blocking_b_6_4_encode_avatar, avatar_data)
+
+        return self.apg._a.build_item_data_elt(avatar_data)
+
+    @ensure_deferred
+    async def items(
+        self,
+        requestor: jid.JID,
+        service: jid.JID,
+        node: str,
+        maxItems: Optional[int],
+        itemIdentifiers: Optional[List[str]],
+        rsm_req: Optional[rsm.RSMRequest]
+    ) -> Tuple[List[domish.Element], Optional[rsm.RSMResponse]]:
+        if not service.user:
+            return [], None
+        ap_account = self.host.plugins["XEP-0106"].unescape(service.user)
+        if ap_account.count("@") != 1:
+            log.warning(f"Invalid AP account used by {requestor}: {ap_account!r}")
+            return [], None
+
+        # cached_node may be pre-filled with some nodes (e.g. attachments nodes),
+        # otherwise it is filled when suitable
+        cached_node = None
+        client = self.apg.client
+        kwargs = {}
+
+        if node == self.apg._pps.subscriptions_node:
+            collection_name = "following"
+            parser = self.ap_following_2_elt
+            kwargs["only_ids"] = True
+            use_cache = False
+        elif node.startswith(self.apg._pps.subscribers_node_prefix):
+            collection_name = "followers"
+            parser = self.ap_follower_2_elt
+            kwargs["only_ids"] = True
+            use_cache = False
+        elif node == self.apg._v.node:
+            # vCard4 request
+            item_elt = await self.generate_v_card(ap_account)
+            return [item_elt], None
+        elif node == self.apg._a.namespace_metadata:
+            item_elt = await self.generate_avatar_metadata(self.apg.client, ap_account)
+            return [item_elt], None
+        elif node == self.apg._a.namespace_data:
+            item_elt = await self.generate_avatar_data(
+                self.apg.client, ap_account, itemIdentifiers
+            )
+            return [item_elt], None
+        elif self.apg._pa.is_attachment_node(node):
+            use_cache = True
+            # we check cache here because we emit an item-not-found error if the node is
+            # not in cache, as we are not dealing with real AP items
+            cached_node = await self.host.memory.storage.get_pubsub_node(
+                client, service, node
+            )
+            if cached_node is None:
+                raise error.StanzaError("item-not-found")
+        else:
+            if node.startswith(self.apg._m.namespace):
+                parser = self.apg.ap_item_2_mb_elt
+            elif node.startswith(self.apg._events.namespace):
+                parser = self.apg.ap_events.ap_item_2_event_elt
+            else:
+                raise error.StanzaError(
+                    "feature-not-implemented",
+                    text=f"AP Gateway {C.APP_VERSION} only supports "
+                    f"{self.apg._m.namespace} node for now"
+                )
+            collection_name = "outbox"
+            use_cache = True
+
+        if use_cache:
+            if cached_node is None:
+                cached_node = await self.host.memory.storage.get_pubsub_node(
+                    client, service, node
+                )
+            # TODO: check if node is synchronised
+            if cached_node is not None:
+                # the node is cached, we return items from cache
+                log.debug(f"node {node!r} from {service} is in cache")
+                pubsub_items, metadata = await self.apg._c.get_items_from_cache(
+                    client, cached_node, maxItems, itemIdentifiers, rsm_request=rsm_req
+                )
+                try:
+                    rsm_resp = rsm.RSMResponse(**metadata["rsm"])
+                except KeyError:
+                    rsm_resp = None
+                return [i.data for i in pubsub_items], rsm_resp
+
+        if itemIdentifiers:
+            items = []
+            for item_id in itemIdentifiers:
+                item_data = await self.apg.ap_get(item_id)
+                item_elt = await parser(item_data)
+                items.append(item_elt)
+            return items, None
+        else:
+            if rsm_req is None:
+                if maxItems is None:
+                    maxItems = 20
+                kwargs.update({
+                    "max_items": maxItems,
+                    "chronological_pagination": False,
+                })
+            else:
+                if len(
+                    [v for v in (rsm_req.after, rsm_req.before, rsm_req.index)
+                     if v is not None]
+                ) > 1:
+                    raise error.StanzaError(
+                        "bad-request",
+                        text="You can't use after, before and index at the same time"
+                    )
+                kwargs.update({"max_items": rsm_req.max})
+                if rsm_req.after is not None:
+                    kwargs["after_id"] = rsm_req.after
+                elif rsm_req.before is not None:
+                    kwargs["chronological_pagination"] = False
+                    if rsm_req.before != "":
+                        kwargs["after_id"] = rsm_req.before
+                elif rsm_req.index is not None:
+                    kwargs["start_index"] = rsm_req.index
+
+            log.info(
+                f"No cache found for node {node} at {service} (AP account {ap_account}), "
+                "using Collection Paging to RSM translation"
+            )
+            if self.apg._m.is_comment_node(node):
+                parent_item = self.apg._m.get_parent_item(node)
+                try:
+                    parent_data = await self.apg.ap_get(parent_item)
+                    collection = await self.apg.ap_get_object(
+                        parent_data.get("object", {}),
+                        "replies"
+                    )
+                except Exception as e:
+                    raise error.StanzaError(
+                        "item-not-found",
+                        text=e
+                    )
+            else:
+                actor_data = await self.apg.get_ap_actor_data_from_account(ap_account)
+                collection = await self.apg.ap_get_object(actor_data, collection_name)
+            if not collection:
+                raise error.StanzaError(
+                    "item-not-found",
+                    text=f"No collection found for node {node!r} (account: {ap_account})"
+                )
+
+            kwargs["parser"] = parser
+            return await self.apg.get_ap_items(collection, **kwargs)
+
+    @ensure_deferred
+    async def retract(self, requestor, service, nodeIdentifier, itemIdentifiers):
+        raise error.StanzaError("forbidden")
+
+    @ensure_deferred
+    async def subscribe(self, requestor, service, nodeIdentifier, subscriber):
+        # TODO: handle comments nodes
+        client = self.apg.client
+        # we use PENDING state for microblog, it will be set to SUBSCRIBED once the Follow
+        # is accepted. Other nodes are directly set to subscribed, their subscriptions
+        # being internal.
+        if nodeIdentifier == self.apg._m.namespace:
+            sub_state = SubscriptionState.PENDING
+        else:
+            sub_state = SubscriptionState.SUBSCRIBED
+        node = await self.host.memory.storage.get_pubsub_node(
+            client, service, nodeIdentifier, with_subscriptions=True
+        )
+        if node is None:
+            node = await self.host.memory.storage.set_pubsub_node(
+                client,
+                service,
+                nodeIdentifier,
+            )
+            subscription = None
+        else:
+            try:
+                subscription = next(
+                    s for s in node.subscriptions
+                    if s.subscriber == requestor.userhostJID()
+                )
+            except StopIteration:
+                subscription = None
+
+        if subscription is None:
+            subscription = PubsubSub(
+                subscriber=requestor.userhostJID(),
+                state=sub_state
+            )
+            node.subscriptions.append(subscription)
+            await self.host.memory.storage.add(node)
+        else:
+            if subscription.state is None:
+                subscription.state = sub_state
+                await self.host.memory.storage.add(node)
+            elif subscription.state == SubscriptionState.SUBSCRIBED:
+                log.info(
+                    f"{requestor.userhostJID()} has already a subscription to {node!r} "
+                    f"at {service}. Doing the request anyway."
+                )
+            elif subscription.state == SubscriptionState.PENDING:
+                log.info(
+                    f"{requestor.userhostJID()} has already a pending subscription to "
+                    f"{node!r} at {service}. Doing the request anyway."
+                )
+                if sub_state != SubscriptionState.PENDING:
+                    subscription.state = sub_state
+                    await self.host.memory.storage.add(node)
+            else:
+                raise exceptions.InternalError(
+                    f"unmanaged subscription state: {subscription.state}"
+                )
+
+        if nodeIdentifier in (self.apg._m.namespace, self.apg._events.namespace):
+            # if we subscribe to microblog or events node, we follow the corresponding
+            # account
+            req_actor_id, recip_actor_id, inbox = await self.get_ap_actor_ids_and_inbox(
+                requestor, service
+            )
+
+            data = self.apg.create_activity("Follow", req_actor_id, recip_actor_id)
+
+            resp = await self.apg.sign_and_post(inbox, req_actor_id, data)
+            if resp.code >= 300:
+                text = await resp.text()
+                raise error.StanzaError("service-unavailable", text=text)
+        return pubsub.Subscription(nodeIdentifier, requestor, "subscribed")
+
+    @ensure_deferred
+    async def unsubscribe(self, requestor, service, nodeIdentifier, subscriber):
+        req_actor_id, recip_actor_id, inbox = await self.get_ap_actor_ids_and_inbox(
+            requestor, service
+        )
+        data = self.apg.create_activity(
+            "Undo",
+            req_actor_id,
+            self.apg.create_activity(
+                "Follow",
+                req_actor_id,
+                recip_actor_id
+            )
+        )
+
+        resp = await self.apg.sign_and_post(inbox, req_actor_id, data)
+        if resp.code >= 300:
+            text = await resp.text()
+            raise error.StanzaError("service-unavailable", text=text)
+
+    def getConfigurationOptions(self):
+        return NODE_OPTIONS
+
+    def getConfiguration(
+        self,
+        requestor: jid.JID,
+        service: jid.JID,
+        nodeIdentifier: str
+    ) -> defer.Deferred:
+        return defer.succeed(NODE_CONFIG_VALUES)
+
+    def getNodeInfo(
+        self,
+        requestor: jid.JID,
+        service: jid.JID,
+        nodeIdentifier: str,
+        pep: bool = False,
+        recipient: Optional[jid.JID] = None
+    ) -> Optional[dict]:
+        if not nodeIdentifier:
+            return None
+        info = {
+            "type": "leaf",
+            "meta-data": NODE_CONFIG
+        }
+        return info

libervia/backend/plugins/plugin_comp_ap_gateway/regex.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+# Libervia ActivityPub Gateway
+# Copyright (C) 2009-2022 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Various Regular Expression for AP gateway"""
+
+import re
+
+## "Signature" header parsing
+
+# those expression have been generated with abnf-to-regex
+# (https://github.com/aas-core-works/abnf-to-regexp)
+
+# the base RFC 7320 ABNF rules come from https://github.com/EricGT/ABNF
+
+# here is the ABNF file used:
+# ---
+# BWS = OWS
+# OWS = *( SP / HTAB )
+# tchar = "!" / "#" / "$" / "%" / "&" / "`" / "*" / "+" / "-" / "." / "^" / "_" / "\'" / "|" / "~" / DIGIT / ALPHA
+# token = 1*tchar
+# sig-param = token BWS "=" BWS ( token / quoted-string )
+# quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE
+# qdtext = HTAB / SP / "!" / %x23-5B ; '#'-'['
+#  / %x5D-7E ; ']'-'~'
+#  / obs-text
+# quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text )
+# obs-text = %x80-FF
+# ---
+
+ows = '[ \t]*'
+bws = f'{ows}'
+obs_text = '[\\x80-\\xff]'
+qdtext = f'([\t !#-\\[\\]-~]|{obs_text})'
+quoted_pair = f'\\\\([\t !-~]|{obs_text})'
+quoted_string = f'"({qdtext}|{quoted_pair})*"'
+tchar = "([!#$%&`*+\\-.^_]|\\\\'|[|~0-9a-zA-Z])"
+token = f'({tchar})+'
+RE_SIG_PARAM = re.compile(
+    f'(?P<key>{token}{bws})={bws}'
+    f'((?P<uq_value>{token})|(?P<quoted_value>{quoted_string}))'
+)
+
+
+## Account/Mention
+
+# FIXME: naive regex, should be approved following webfinger, but popular implementations
+#   such as Mastodon use a very restricted subset
+RE_ACCOUNT = re.compile(r"[a-zA-Z0-9._-]+@[a-zA-Z0-9-]+.[a-zA-Z0-9-]+")
+RE_MENTION = re.compile(rf"(?<!\w)@{RE_ACCOUNT.pattern}\b")

libervia/backend/plugins/plugin_comp_file_sharing.py

@@ -0,0 +1,884 @@
+#!/usr/bin/env python3
+
+# Libervia File Sharing component
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import os.path
+import mimetypes
+import tempfile
+from functools import partial
+import shortuuid
+import unicodedata
+from urllib.parse import urljoin, urlparse, quote, unquote
+from pathlib import Path
+from libervia.backend.core.i18n import _, D_
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core import exceptions
+from libervia.backend.core.log import getLogger
+from libervia.backend.tools import stream
+from libervia.backend.tools import video
+from libervia.backend.tools.utils import ensure_deferred
+from libervia.backend.tools.common import regex
+from libervia.backend.tools.common import uri
+from libervia.backend.tools.common import files_utils
+from libervia.backend.tools.common import utils
+from libervia.backend.tools.common import tls
+from twisted.internet import defer, reactor
+from twisted.words.protocols.jabber import error
+from twisted.web import server, resource, static, http
+from wokkel import pubsub
+from wokkel import generic
+
+
+log = getLogger(__name__)
+
+
+PLUGIN_INFO = {
+    C.PI_NAME: "File sharing component",
+    C.PI_IMPORT_NAME: "file-sharing",
+    C.PI_MODES: [C.PLUG_MODE_COMPONENT],
+    C.PI_TYPE: C.PLUG_TYPE_ENTRY_POINT,
+    C.PI_PROTOCOLS: [],
+    C.PI_DEPENDENCIES: [
+        "FILE",
+        "FILE_SHARING_MANAGEMENT",
+        "XEP-0106",
+        "XEP-0234",
+        "XEP-0260",
+        "XEP-0261",
+        "XEP-0264",
+        "XEP-0329",
+        "XEP-0363",
+    ],
+    C.PI_RECOMMENDATIONS: [],
+    C.PI_MAIN: "FileSharing",
+    C.PI_HANDLER: C.BOOL_TRUE,
+    C.PI_DESCRIPTION: _("""Component hosting and sharing files"""),
+}
+
+HASH_ALGO = "sha-256"
+NS_COMMENTS = "org.salut-a-toi.comments"
+NS_FS_AFFILIATION = "org.salut-a-toi.file-sharing-affiliation"
+COMMENT_NODE_PREFIX = "org.salut-a-toi.file_comments/"
+# Directory used to buffer request body (i.e. file in case of PUT) we use more than one @
+# there, to be sure than it's not conflicting with a JID
+TMP_BUFFER_DIR = "@@tmp@@"
+OVER_QUOTA_TXT = D_(
+    "You are over quota, your maximum allowed size is {quota} and you are already using "
+    "{used_space}, you can't upload {file_size} more."
+)
+
+HTTP_VERSION = unicodedata.normalize(
+    'NFKD',
+    f"{C.APP_NAME} file sharing {C.APP_VERSION}"
+)
+
+
+class HTTPFileServer(resource.Resource):
+    isLeaf = True
+
+    def errorPage(self, request, code):
+        request.setResponseCode(code)
+        if code == http.BAD_REQUEST:
+            brief = 'Bad Request'
+            details = "Your request is invalid"
+        elif code == http.FORBIDDEN:
+            brief = 'Forbidden'
+            details = "You're not allowed to use this resource"
+        elif code == http.NOT_FOUND:
+            brief = 'Not Found'
+            details = "No resource found at this URL"
+        else:
+            brief = 'Error'
+            details = "This resource can't be used"
+            log.error(f"Unexpected return code used: {code}")
+        log.warning(
+            f'Error returned while trying to access url {request.uri.decode()}: '
+            f'"{brief}" ({code}): {details}'
+        )
+
+        return resource.ErrorPage(code, brief, details).render(request)
+
+    def get_disposition_type(self, media_type, media_subtype):
+        if media_type in ('image', 'video'):
+            return 'inline'
+        elif media_type == 'application' and media_subtype == 'pdf':
+            return 'inline'
+        else:
+            return 'attachment'
+
+    def render(self, request):
+        request.setHeader("server", HTTP_VERSION)
+        request.setHeader("Access-Control-Allow-Origin", "*")
+        request.setHeader("Access-Control-Allow-Methods", "OPTIONS, HEAD, GET, PUT")
+        request.setHeader(
+            "Access-Control-Allow-Headers",
+            "Content-Type, Range, Xmpp-File-Path, Xmpp-File-No-Http")
+        request.setHeader("Access-Control-Allow-Credentials", "true")
+        request.setHeader("Accept-Ranges", "bytes")
+
+        request.setHeader(
+            "Access-Control-Expose-Headers",
+            "Date, Content-Length, Content-Range")
+        return super().render(request)
+
+    def render_options(self, request):
+        request.setResponseCode(http.OK)
+        return b""
+
+    def render_GET(self, request):
+        try:
+            request.upload_data
+        except exceptions.DataError:
+            return self.errorPage(request, http.NOT_FOUND)
+
+        defer.ensureDeferred(self.render_get(request))
+        return server.NOT_DONE_YET
+
+    async def render_get(self, request):
+        try:
+            upload_id, filename = request.upload_data
+        except exceptions.DataError:
+            request.write(self.errorPage(request, http.FORBIDDEN))
+            request.finish()
+            return
+        found_files = await request.file_sharing.host.memory.get_files(
+            client=None, peer_jid=None, perms_to_check=None, public_id=upload_id)
+        if not found_files:
+            request.write(self.errorPage(request, http.NOT_FOUND))
+            request.finish()
+            return
+        if len(found_files) > 1:
+            log.error(f"more that one files found for public id {upload_id!r}")
+
+        found_file = found_files[0]
+        file_path = request.file_sharing.files_path/found_file['file_hash']
+        file_res = static.File(file_path)
+        file_res.type = f'{found_file["media_type"]}/{found_file["media_subtype"]}'
+        file_res.encoding = file_res.contentEncodings.get(Path(found_file['name']).suffix)
+        disp_type = self.get_disposition_type(
+            found_file['media_type'], found_file['media_subtype'])
+        # the URL is percent encoded, and not all browsers/tools unquote the file name,
+        # thus we add a content disposition header
+        request.setHeader(
+            'Content-Disposition',
+            f"{disp_type}; filename*=UTF-8''{quote(found_file['name'])}"
+        )
+        # cf. https://xmpp.org/extensions/xep-0363.html#server
+        request.setHeader(
+            'Content-Security-Policy',
+            "default-src 'none'; frame-ancestors 'none';"
+        )
+        ret = file_res.render(request)
+        if ret != server.NOT_DONE_YET:
+            # HEAD returns directly the result (while GET use a produced)
+            request.write(ret)
+            request.finish()
+
+    def render_PUT(self, request):
+        defer.ensureDeferred(self.render_put(request))
+        return server.NOT_DONE_YET
+
+    async def render_put(self, request):
+        try:
+            client, upload_request = request.upload_request_data
+            upload_id, filename = request.upload_data
+        except AttributeError:
+            request.write(self.errorPage(request, http.BAD_REQUEST))
+            request.finish()
+            return
+
+        # at this point request is checked and file is buffered, we can store it
+        # we close the content here, before registering the file
+        request.content.close()
+        tmp_file_path = Path(request.content.name)
+        request.content = None
+
+        # the 2 following headers are not standard, but useful in the context of file
+        # sharing with HTTP Upload: first one allow uploader to specify the path
+        # and second one will disable public exposure of the file through HTTP
+        path = request.getHeader("Xmpp-File-Path")
+        if path:
+            path = unquote(path)
+        else:
+            path =  "/uploads"
+        if request.getHeader("Xmpp-File-No-Http") is not None:
+            public_id = None
+        else:
+            public_id = upload_id
+
+        file_data = {
+            "name": unquote(upload_request.filename),
+            "mime_type": upload_request.content_type,
+            "size": upload_request.size,
+            "path": path
+        }
+
+        await request.file_sharing.register_received_file(
+            client, upload_request.from_, file_data, tmp_file_path,
+            public_id=public_id,
+        )
+
+        request.setResponseCode(http.CREATED)
+        request.finish()
+
+
+class FileSharingRequest(server.Request):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._upload_data = None
+
+    @property
+    def upload_data(self):
+        """A tuple with upload_id and filename retrieved from requested path"""
+        if self._upload_data is not None:
+            return self._upload_data
+
+        # self.path is not available if we are early in the request (e.g. when gotLength
+        # is called), in which case channel._path must be used. On the other hand, when
+        # render_[VERB] is called, only self.path is available
+        path = self.channel._path if self.path is None else self.path
+        # we normalise the path
+        path = urlparse(path.decode()).path
+        try:
+            __, upload_id, filename = path.split('/')
+        except ValueError:
+            raise exceptions.DataError("no enought path elements")
+        if len(upload_id) < 10:
+            raise exceptions.DataError(f"invalid upload ID received for a PUT: {upload_id!r}")
+
+        self._upload_data = (upload_id, filename)
+        return self._upload_data
+
+    @property
+    def file_sharing(self):
+        return self.channel.site.file_sharing
+
+    @property
+    def file_tmp_dir(self):
+        return self.channel.site.file_tmp_dir
+
+    def refuse_request(self):
+        if self.content is not None:
+            self.content.close()
+        self.content = open(os.devnull, 'w+b')
+        self.channel._respondToBadRequestAndDisconnect()
+
+    def gotLength(self, length):
+        if self.channel._command.decode().upper() == 'PUT':
+            # for PUT we check early if upload_id is fine, to avoid buffering a file we'll refuse
+            # we buffer the file in component's TMP_BUFFER_DIR, so we just have to rename it at the end
+            try:
+                upload_id, filename = self.upload_data
+            except exceptions.DataError as e:
+                log.warning(f"Invalid PUT request, we stop here: {e}")
+                return self.refuse_request()
+            try:
+                client, upload_request, timer = self.file_sharing.expected_uploads.pop(upload_id)
+            except KeyError:
+                log.warning(f"unknown (expired?) upload ID received for a PUT: {upload_id!r}")
+                return self.refuse_request()
+
+            if not timer.active:
+                log.warning(f"upload id {upload_id!r} used for a PUT, but it is expired")
+                return self.refuse_request()
+
+            timer.cancel()
+
+            if upload_request.filename != filename:
+                log.warning(
+                    f"invalid filename for PUT (upload id: {upload_id!r}, URL: {self.channel._path.decode()}). Original "
+                    f"{upload_request.filename!r} doesn't match {filename!r}"
+                )
+                return self.refuse_request()
+
+            self.upload_request_data = (client, upload_request)
+
+            file_tmp_path = files_utils.get_unique_name(
+                self.file_tmp_dir/upload_id)
+
+            self.content = open(file_tmp_path, 'w+b')
+        else:
+            return super().gotLength(length)
+
+
+class FileSharingSite(server.Site):
+    requestFactory = FileSharingRequest
+
+    def __init__(self, file_sharing):
+        self.file_sharing = file_sharing
+        self.file_tmp_dir = file_sharing.host.get_local_path(
+            None, C.FILES_TMP_DIR, TMP_BUFFER_DIR, component=True
+        )
+        for old_file in self.file_tmp_dir.iterdir():
+            log.debug(f"purging old buffer file at {old_file}")
+            old_file.unlink()
+        super().__init__(HTTPFileServer())
+
+    def getContentFile(self, length):
+        file_tmp_path = self.file_tmp_dir/shortuuid.uuid()
+        return open(file_tmp_path, 'w+b')
+
+
+class FileSharing:
+
+    def __init__(self, host):
+        self.host = host
+        self.initialised = False
+
+    def init(self):
+        # we init once on first component connection,
+        # there is not need to init this plugin if not component use it
+        # TODO: this plugin should not be loaded at all if no component uses it
+        #   and should be loaded dynamically as soon as a suitable profile is created
+        if self.initialised:
+            return
+        self.initialised = True
+        log.info(_("File Sharing initialization"))
+        self._f = self.host.plugins["FILE"]
+        self._jf = self.host.plugins["XEP-0234"]
+        self._h = self.host.plugins["XEP-0300"]
+        self._t = self.host.plugins["XEP-0264"]
+        self._hu = self.host.plugins["XEP-0363"]
+        self._hu.register_handler(self._on_http_upload)
+        self.host.trigger.add("FILE_getDestDir", self._get_dest_dir_trigger)
+        self.host.trigger.add(
+            "XEP-0234_fileSendingRequest", self._file_sending_request_trigger, priority=1000
+        )
+        self.host.trigger.add("XEP-0234_buildFileElement", self._add_file_metadata_elts)
+        self.host.trigger.add("XEP-0234_parseFileElement", self._get_file_metadata_elts)
+        self.host.trigger.add("XEP-0329_compGetFilesFromNode", self._add_file_metadata)
+        self.host.trigger.add(
+            "XEP-0329_compGetFilesFromNode_build_directory",
+            self._add_directory_metadata_elts)
+        self.host.trigger.add(
+            "XEP-0329_parseResult_directory",
+            self._get_directory_metadata_elts)
+        self.files_path = self.host.get_local_path(None, C.FILES_DIR)
+        self.http_port = int(self.host.memory.config_get(
+            'component file-sharing', 'http_upload_port', 8888))
+        connection_type = self.host.memory.config_get(
+            'component file-sharing', 'http_upload_connection_type', 'https')
+        if connection_type not in ('http', 'https'):
+            raise exceptions.ConfigError(
+                'bad http_upload_connection_type, you must use one of "http" or "https"'
+            )
+        self.server = FileSharingSite(self)
+        self.expected_uploads = {}
+        if connection_type == 'http':
+            reactor.listenTCP(self.http_port, self.server)
+        else:
+            options = tls.get_options_from_config(
+                self.host.memory.config, "component file-sharing")
+            tls.tls_options_check(options)
+            context_factory = tls.get_tls_context_factory(options)
+            reactor.listenSSL(self.http_port, self.server, context_factory)
+
+    def get_handler(self, client):
+        return Comments_handler(self)
+
+    def profile_connecting(self, client):
+        # we activate HTTP upload
+        client.enabled_features.add("XEP-0363")
+
+        self.init()
+        public_base_url = self.host.memory.config_get(
+            'component file-sharing', 'http_upload_public_facing_url')
+        if public_base_url is None:
+            client._file_sharing_base_url = f"https://{client.host}:{self.http_port}"
+        else:
+            client._file_sharing_base_url = public_base_url
+        path = client.file_tmp_dir = os.path.join(
+            self.host.memory.config_get("", "local_dir"),
+            C.FILES_TMP_DIR,
+            regex.path_escape(client.profile),
+        )
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+    def get_quota(self, client, entity):
+        """Return maximum size allowed for all files for entity"""
+        quotas = self.host.memory.config_get("component file-sharing", "quotas_json", {})
+        if self.host.memory.is_admin_jid(entity):
+            quota = quotas.get("admins")
+        else:
+            try:
+                quota = quotas["jids"][entity.userhost()]
+            except KeyError:
+                quota = quotas.get("users")
+        return None if quota is None else utils.parse_size(quota)
+
+    async def generate_thumbnails(self, extra: dict, image_path: Path):
+        thumbnails = extra.setdefault(C.KEY_THUMBNAILS, [])
+        for max_thumb_size in self._t.SIZES:
+            try:
+                thumb_size, thumb_id = await self._t.generate_thumbnail(
+                    image_path,
+                    max_thumb_size,
+                    #  we keep thumbnails for 6 months
+                    60 * 60 * 24 * 31 * 6,
+                )
+            except Exception as e:
+                log.warning(_("Can't create thumbnail: {reason}").format(reason=e))
+                break
+            thumbnails.append({"id": thumb_id, "size": thumb_size})
+
+    async def register_received_file(
+            self, client, peer_jid, file_data, file_path, public_id=None, extra=None):
+        """Post file reception tasks
+
+        once file is received, this method create hash/thumbnails if necessary
+        move the file to the right location, and create metadata entry in database
+        """
+        name = file_data["name"]
+        if extra is None:
+            extra = {}
+
+        mime_type = file_data.get("mime_type")
+        if not mime_type or mime_type == "application/octet-stream":
+            mime_type = mimetypes.guess_type(name)[0]
+
+        is_image = mime_type is not None and mime_type.startswith("image")
+        is_video = mime_type is not None and mime_type.startswith("video")
+
+        if file_data.get("hash_algo") == HASH_ALGO:
+            log.debug(_("Reusing already generated hash"))
+            file_hash = file_data["hash_hasher"].hexdigest()
+        else:
+            hasher = self._h.get_hasher(HASH_ALGO)
+            with file_path.open('rb') as f:
+                file_hash = await self._h.calculate_hash(f, hasher)
+        final_path = self.files_path/file_hash
+
+        if final_path.is_file():
+            log.debug(
+                "file [{file_hash}] already exists, we can remove temporary one".format(
+                    file_hash=file_hash
+                )
+            )
+            file_path.unlink()
+        else:
+            file_path.rename(final_path)
+            log.debug(
+                "file [{file_hash}] moved to {files_path}".format(
+                    file_hash=file_hash, files_path=self.files_path
+                )
+            )
+
+        if is_image:
+            await self.generate_thumbnails(extra, final_path)
+        elif is_video:
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                thumb_path = Path(tmp_dir) / "thumbnail.jpg"
+                try:
+                    await video.get_thumbnail(final_path, thumb_path)
+                except Exception as e:
+                    log.warning(_("Can't get thumbnail for {final_path}: {e}").format(
+                        final_path=final_path, e=e))
+                else:
+                    await self.generate_thumbnails(extra, thumb_path)
+
+        await self.host.memory.set_file(
+            client,
+            name=name,
+            version="",
+            file_hash=file_hash,
+            hash_algo=HASH_ALGO,
+            size=file_data["size"],
+            path=file_data.get("path"),
+            namespace=file_data.get("namespace"),
+            mime_type=mime_type,
+            public_id=public_id,
+            owner=peer_jid,
+            extra=extra,
+        )
+
+    async def _get_dest_dir_trigger(
+        self, client, peer_jid, transfer_data, file_data, stream_object
+    ):
+        """This trigger accept file sending request, and store file locally"""
+        if not client.is_component:
+            return True, None
+        # client._file_sharing_allowed_hosts is set in plugin XEP-0329
+        if peer_jid.host not in client._file_sharing_allowed_hosts:
+            raise error.StanzaError("forbidden")
+        assert stream_object
+        assert "stream_object" not in transfer_data
+        assert C.KEY_PROGRESS_ID in file_data
+        filename = file_data["name"]
+        assert filename and not "/" in filename
+        quota = self.get_quota(client, peer_jid)
+        if quota is not None:
+            used_space = await self.host.memory.file_get_used_space(client, peer_jid)
+
+            if (used_space + file_data["size"]) > quota:
+                raise error.StanzaError(
+                    "not-acceptable",
+                    text=OVER_QUOTA_TXT.format(
+                        quota=utils.get_human_size(quota),
+                        used_space=utils.get_human_size(used_space),
+                        file_size=utils.get_human_size(file_data['size'])
+                    )
+                )
+        file_tmp_dir = self.host.get_local_path(
+            None, C.FILES_TMP_DIR, peer_jid.userhost(), component=True
+        )
+        file_tmp_path = file_data['file_path'] = files_utils.get_unique_name(
+            file_tmp_dir/filename)
+
+        transfer_data["finished_d"].addCallback(
+            lambda __: defer.ensureDeferred(
+                self.register_received_file(client, peer_jid, file_data, file_tmp_path)
+            )
+        )
+
+        self._f.open_file_write(
+            client, file_tmp_path, transfer_data, file_data, stream_object
+        )
+        return False, True
+
+    async def _retrieve_files(
+        self, client, session, content_data, content_name, file_data, file_elt
+    ):
+        """This method retrieve a file on request, and send if after checking permissions"""
+        peer_jid = session["peer_jid"]
+        if session['local_jid'].user:
+            owner = client.get_owner_from_jid(session['local_jid'])
+        else:
+            owner = peer_jid
+        try:
+            found_files = await self.host.memory.get_files(
+                client,
+                peer_jid=peer_jid,
+                name=file_data.get("name"),
+                file_hash=file_data.get("file_hash"),
+                hash_algo=file_data.get("hash_algo"),
+                path=file_data.get("path"),
+                namespace=file_data.get("namespace"),
+                owner=owner,
+            )
+        except exceptions.NotFound:
+            found_files = None
+        except exceptions.PermissionError:
+            log.warning(
+                _("{peer_jid} is trying to access an unauthorized file: {name}").format(
+                    peer_jid=peer_jid, name=file_data.get("name")
+                )
+            )
+            return False
+
+        if not found_files:
+            log.warning(
+                _("no matching file found ({file_data})").format(file_data=file_data)
+            )
+            return False
+
+        # we only use the first found file
+        found_file = found_files[0]
+        if found_file['type'] != C.FILE_TYPE_FILE:
+            raise TypeError("a file was expected, type is {type_}".format(
+                type_=found_file['type']))
+        file_hash = found_file["file_hash"]
+        file_path = self.files_path / file_hash
+        file_data["hash_hasher"] = hasher = self._h.get_hasher(found_file["hash_algo"])
+        size = file_data["size"] = found_file["size"]
+        file_data["file_hash"] = file_hash
+        file_data["hash_algo"] = found_file["hash_algo"]
+
+        # we complete file_elt so peer can have some details on the file
+        if "name" not in file_data:
+            file_elt.addElement("name", content=found_file["name"])
+        file_elt.addElement("size", content=str(size))
+        content_data["stream_object"] = stream.FileStreamObject(
+            self.host,
+            client,
+            file_path,
+            uid=self._jf.get_progress_id(session, content_name),
+            size=size,
+            data_cb=lambda data: hasher.update(data),
+        )
+        return True
+
+    def _file_sending_request_trigger(
+        self, client, session, content_data, content_name, file_data, file_elt
+    ):
+        if not client.is_component:
+            return True, None
+        else:
+            return (
+                False,
+                defer.ensureDeferred(self._retrieve_files(
+                    client, session, content_data, content_name, file_data, file_elt
+                )),
+            )
+
+    ## HTTP Upload ##
+
+    def _purge_slot(self, upload_id):
+        try:
+            del self.expected_uploads[upload_id]
+        except KeyError:
+            log.error(f"trying to purge an inexisting upload slot ({upload_id})")
+
+    async def _on_http_upload(self, client, request):
+        # filename should be already cleaned, but it's better to double check
+        assert '/' not in request.filename
+        # client._file_sharing_allowed_hosts is set in plugin XEP-0329
+        if request.from_.host not in client._file_sharing_allowed_hosts:
+            raise error.StanzaError("forbidden")
+
+        quota = self.get_quota(client, request.from_)
+        if quota is not None:
+            used_space = await self.host.memory.file_get_used_space(client, request.from_)
+
+            if (used_space + request.size) > quota:
+                raise error.StanzaError(
+                    "not-acceptable",
+                    text=OVER_QUOTA_TXT.format(
+                        quota=utils.get_human_size(quota),
+                        used_space=utils.get_human_size(used_space),
+                        file_size=utils.get_human_size(request.size)
+                    ),
+                    appCondition = self._hu.get_file_too_large_elt(max(quota - used_space, 0))
+                )
+
+        upload_id = shortuuid.ShortUUID().random(length=30)
+        assert '/' not in upload_id
+        timer = reactor.callLater(30, self._purge_slot, upload_id)
+        self.expected_uploads[upload_id] = (client, request, timer)
+        url = urljoin(client._file_sharing_base_url, f"{upload_id}/{request.filename}")
+        slot = self._hu.Slot(
+            put=url,
+            get=url,
+            headers=[],
+        )
+        return slot
+
+    ## metadata triggers ##
+
+    def _add_file_metadata_elts(self, client, file_elt, extra_args):
+        # affiliation
+        affiliation = extra_args.get('affiliation')
+        if affiliation is not None:
+            file_elt.addElement((NS_FS_AFFILIATION, "affiliation"), content=affiliation)
+
+        # comments
+        try:
+            comments_url = extra_args.pop("comments_url")
+        except KeyError:
+            return
+
+        comment_elt = file_elt.addElement((NS_COMMENTS, "comments"), content=comments_url)
+
+        try:
+            count = len(extra_args["extra"]["comments"])
+        except KeyError:
+            count = 0
+
+        comment_elt["count"] = str(count)
+        return True
+
+    def _get_file_metadata_elts(self, client, file_elt, file_data):
+        # affiliation
+        try:
+            affiliation_elt = next(file_elt.elements(NS_FS_AFFILIATION, "affiliation"))
+        except StopIteration:
+            pass
+        else:
+            file_data["affiliation"] = str(affiliation_elt)
+
+        # comments
+        try:
+            comments_elt = next(file_elt.elements(NS_COMMENTS, "comments"))
+        except StopIteration:
+            pass
+        else:
+            file_data["comments_url"] = str(comments_elt)
+            file_data["comments_count"] = comments_elt["count"]
+        return True
+
+    def _add_file_metadata(
+            self, client, iq_elt, iq_result_elt, owner, node_path, files_data):
+        for file_data in files_data:
+            file_data["comments_url"] = uri.build_xmpp_uri(
+                "pubsub",
+                path=client.jid.full(),
+                node=COMMENT_NODE_PREFIX + file_data["id"],
+            )
+        return True
+
+    def _add_directory_metadata_elts(
+            self, client, file_data, directory_elt, owner, node_path):
+        affiliation = file_data.get('affiliation')
+        if affiliation is not None:
+            directory_elt.addElement(
+                (NS_FS_AFFILIATION, "affiliation"),
+                content=affiliation
+            )
+
+    def _get_directory_metadata_elts(
+            self, client, elt, file_data):
+        try:
+            affiliation_elt = next(elt.elements(NS_FS_AFFILIATION, "affiliation"))
+        except StopIteration:
+            pass
+        else:
+            file_data['affiliation'] = str(affiliation_elt)
+
+
+class Comments_handler(pubsub.PubSubService):
+    """This class is a minimal Pubsub service handling virtual nodes for comments"""
+
+    def __init__(self, plugin_parent):
+        super(Comments_handler, self).__init__()
+        self.host = plugin_parent.host
+        self.plugin_parent = plugin_parent
+        self.discoIdentity = {
+            "category": "pubsub",
+            "type": "virtual",  # FIXME: non standard, here to avoid this service being considered as main pubsub one
+            "name": "files commenting service",
+        }
+
+    def _get_file_id(self, nodeIdentifier):
+        if not nodeIdentifier.startswith(COMMENT_NODE_PREFIX):
+            raise error.StanzaError("item-not-found")
+        file_id = nodeIdentifier[len(COMMENT_NODE_PREFIX) :]
+        if not file_id:
+            raise error.StanzaError("item-not-found")
+        return file_id
+
+    async def get_file_data(self, requestor, nodeIdentifier):
+        file_id = self._get_file_id(nodeIdentifier)
+        try:
+            files = await self.host.memory.get_files(self.parent, requestor, file_id)
+        except (exceptions.NotFound, exceptions.PermissionError):
+            # we don't differenciate between NotFound and PermissionError
+            # to avoid leaking information on existing files
+            raise error.StanzaError("item-not-found")
+        if not files:
+            raise error.StanzaError("item-not-found")
+        if len(files) > 1:
+            raise error.InternalError("there should be only one file")
+        return files[0]
+
+    def comments_update(self, extra, new_comments, peer_jid):
+        """update comments (replace or insert new_comments)
+
+        @param extra(dict): extra data to update
+        @param new_comments(list[tuple(unicode, unicode, unicode)]): comments to update or insert
+        @param peer_jid(unicode, None): bare jid of the requestor, or None if request is done by owner
+        """
+        current_comments = extra.setdefault("comments", [])
+        new_comments_by_id = {c[0]: c for c in new_comments}
+        updated = []
+        # we now check every current comment, to see if one id in new ones
+        # exist, in which case we must update
+        for idx, comment in enumerate(current_comments):
+            comment_id = comment[0]
+            if comment_id in new_comments_by_id:
+                # a new comment has an existing id, update is requested
+                if peer_jid and comment[1] != peer_jid:
+                    # requestor has not the right to modify the comment
+                    raise exceptions.PermissionError
+                # we replace old_comment with updated one
+                new_comment = new_comments_by_id[comment_id]
+                current_comments[idx] = new_comment
+                updated.append(new_comment)
+
+        # we now remove every updated comments, to only keep
+        # the ones to insert
+        for comment in updated:
+            new_comments.remove(comment)
+
+        current_comments.extend(new_comments)
+
+    def comments_delete(self, extra, comments):
+        try:
+            comments_dict = extra["comments"]
+        except KeyError:
+            return
+        for comment in comments:
+            try:
+                comments_dict.remove(comment)
+            except ValueError:
+                continue
+
+    def _get_from(self, item_elt):
+        """retrieve publisher of an item
+
+        @param item_elt(domish.element): <item> element
+        @return (unicode): full jid as string
+        """
+        iq_elt = item_elt
+        while iq_elt.parent != None:
+            iq_elt = iq_elt.parent
+        return iq_elt["from"]
+
+    @ensure_deferred
+    async def publish(self, requestor, service, nodeIdentifier, items):
+        #  we retrieve file a first time to check authorisations
+        file_data = await self.get_file_data(requestor, nodeIdentifier)
+        file_id = file_data["id"]
+        comments = [(item["id"], self._get_from(item), item.toXml()) for item in items]
+        if requestor.userhostJID() == file_data["owner"]:
+            peer_jid = None
+        else:
+            peer_jid = requestor.userhost()
+        update_cb = partial(self.comments_update, new_comments=comments, peer_jid=peer_jid)
+        try:
+            await self.host.memory.file_update(file_id, "extra", update_cb)
+        except exceptions.PermissionError:
+            raise error.StanzaError("not-authorized")
+
+    @ensure_deferred
+    async def items(self, requestor, service, nodeIdentifier, maxItems, itemIdentifiers):
+        file_data = await self.get_file_data(requestor, nodeIdentifier)
+        comments = file_data["extra"].get("comments", [])
+        if itemIdentifiers:
+            return [generic.parseXml(c[2]) for c in comments if c[0] in itemIdentifiers]
+        else:
+            return [generic.parseXml(c[2]) for c in comments]
+
+    @ensure_deferred
+    async def retract(self, requestor, service, nodeIdentifier, itemIdentifiers):
+        file_data = await self.get_file_data(requestor, nodeIdentifier)
+        file_id = file_data["id"]
+        try:
+            comments = file_data["extra"]["comments"]
+        except KeyError:
+            raise error.StanzaError("item-not-found")
+
+        to_remove = []
+        for comment in comments:
+            comment_id = comment[0]
+            if comment_id in itemIdentifiers:
+                to_remove.append(comment)
+                itemIdentifiers.remove(comment_id)
+                if not itemIdentifiers:
+                    break
+
+        if itemIdentifiers:
+            # not all items have been to_remove, we can't continue
+            raise error.StanzaError("item-not-found")
+
+        if requestor.userhostJID() != file_data["owner"]:
+            if not all([c[1] == requestor.userhost() for c in to_remove]):
+                raise error.StanzaError("not-authorized")
+
+        remove_cb = partial(self.comments_delete, comments=to_remove)
+        await self.host.memory.file_update(file_id, "extra", remove_cb)

libervia/backend/plugins/plugin_comp_file_sharing_management.py

@@ -0,0 +1,483 @@
+#!/usr/bin/env python3
+
+# Libervia plugin to manage file sharing component through ad-hoc commands
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os.path
+from functools import partial
+from wokkel import data_form
+from twisted.internet import defer
+from twisted.words.protocols.jabber import jid
+from libervia.backend.core.i18n import _, D_
+from libervia.backend.core import exceptions
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.log import getLogger
+from libervia.backend.tools.common import utils
+
+log = getLogger(__name__)
+
+
+PLUGIN_INFO = {
+    C.PI_NAME: "File Sharing Management",
+    C.PI_IMPORT_NAME: "FILE_SHARING_MANAGEMENT",
+    C.PI_MODES: [C.PLUG_MODE_COMPONENT],
+    C.PI_TYPE: "EXP",
+    C.PI_PROTOCOLS: [],
+    C.PI_DEPENDENCIES: ["XEP-0050", "XEP-0264"],
+    C.PI_RECOMMENDATIONS: [],
+    C.PI_MAIN: "FileSharingManagement",
+    C.PI_HANDLER: "no",
+    C.PI_DESCRIPTION: _(
+        "Experimental handling of file management for file sharing. This plugins allows "
+        "to change permissions of stored files/directories or remove them."
+    ),
+}
+
+NS_FILE_MANAGEMENT = "https://salut-a-toi.org/protocol/file-management:0"
+NS_FILE_MANAGEMENT_PERM = "https://salut-a-toi.org/protocol/file-management:0#perm"
+NS_FILE_MANAGEMENT_DELETE = "https://salut-a-toi.org/protocol/file-management:0#delete"
+NS_FILE_MANAGEMENT_THUMB = "https://salut-a-toi.org/protocol/file-management:0#thumb"
+NS_FILE_MANAGEMENT_QUOTA = "https://salut-a-toi.org/protocol/file-management:0#quota"
+
+
+class WorkflowError(Exception):
+    """Raised when workflow can't be completed"""
+
+    def __init__(self, err_args):
+        """
+        @param err_args(tuple): arguments to return to finish the command workflow
+        """
+        Exception.__init__(self)
+        self.err_args = err_args
+
+
+class FileSharingManagement(object):
+    # This is a temporary way (Q&D) to handle stored files, a better way (using pubsub
+    # syntax?) should be elaborated and proposed as a standard.
+
+    def __init__(self, host):
+        log.info(_("File Sharing Management plugin initialization"))
+        self.host = host
+        self._c = host.plugins["XEP-0050"]
+        self._t = host.plugins["XEP-0264"]
+        self.files_path = host.get_local_path(None, C.FILES_DIR)
+        host.bridge.add_method(
+            "file_sharing_delete",
+            ".plugin",
+            in_sign="ssss",
+            out_sign="",
+            method=self._delete,
+            async_=True,
+        )
+
+    def profile_connected(self, client):
+        self._c.add_ad_hoc_command(
+            client, self._on_change_file, "Change Permissions of File(s)",
+            node=NS_FILE_MANAGEMENT_PERM,
+            allowed_magics=C.ENTITY_ALL,
+        )
+        self._c.add_ad_hoc_command(
+            client, self._on_delete_file, "Delete File(s)",
+            node=NS_FILE_MANAGEMENT_DELETE,
+            allowed_magics=C.ENTITY_ALL,
+        )
+        self._c.add_ad_hoc_command(
+            client, self._on_gen_thumbnails, "Generate Thumbnails",
+            node=NS_FILE_MANAGEMENT_THUMB,
+            allowed_magics=C.ENTITY_ALL,
+        )
+        self._c.add_ad_hoc_command(
+            client, self._on_quota, "Get Quota",
+            node=NS_FILE_MANAGEMENT_QUOTA,
+            allowed_magics=C.ENTITY_ALL,
+        )
+
+    def _delete(self, service_jid_s, path, namespace, profile):
+        client = self.host.get_client(profile)
+        service_jid = jid.JID(service_jid_s) if service_jid_s else None
+        return defer.ensureDeferred(self._c.sequence(
+            client,
+            [{"path": path, "namespace": namespace}, {"confirm": True}],
+            NS_FILE_MANAGEMENT_DELETE,
+            service_jid,
+        ))
+
+    def _err(self, reason):
+        """Helper method to get argument to return for error
+
+        workflow will be interrupted with an error note
+        @param reason(unicode): reason of the error
+        @return (tuple): arguments to use in defer.returnValue
+        """
+        status = self._c.STATUS.COMPLETED
+        payload = None
+        note = (self._c.NOTE.ERROR, reason)
+        return payload, status, None, note
+
+    def _get_root_args(self):
+        """Create the form to select the file to use
+
+        @return (tuple): arguments to use in defer.returnValue
+        """
+        status = self._c.STATUS.EXECUTING
+        form = data_form.Form("form", title="File Management",
+                              formNamespace=NS_FILE_MANAGEMENT)
+
+        field = data_form.Field(
+            "text-single", "path", required=True
+        )
+        form.addField(field)
+
+        field = data_form.Field(
+            "text-single", "namespace", required=False
+        )
+        form.addField(field)
+
+        payload = form.toElement()
+        return payload, status, None, None
+
+    async def _get_file_data(self, client, session_data, command_form):
+        """Retrieve field requested in root form
+
+        "found_file" will also be set in session_data
+        @param command_form(data_form.Form): response to root form
+        @return (D(dict)): found file data
+        @raise WorkflowError: something is wrong
+        """
+        fields = command_form.fields
+        try:
+            path = fields['path'].value.strip()
+            namespace = fields['namespace'].value or None
+        except KeyError:
+            self._c.ad_hoc_error(self._c.ERROR.BAD_PAYLOAD)
+
+        if not path:
+            self._c.ad_hoc_error(self._c.ERROR.BAD_PAYLOAD)
+
+        requestor = session_data['requestor']
+        requestor_bare = requestor.userhostJID()
+        path = path.rstrip('/')
+        parent_path, basename = os.path.split(path)
+
+        # TODO: if parent_path and basename are empty, we ask for root directory
+        #       this must be managed
+
+        try:
+            found_files = await self.host.memory.get_files(
+                client, requestor_bare, path=parent_path, name=basename,
+                namespace=namespace)
+            found_file = found_files[0]
+        except (exceptions.NotFound, IndexError):
+            raise WorkflowError(self._err(_("file not found")))
+        except exceptions.PermissionError:
+            raise WorkflowError(self._err(_("forbidden")))
+
+        if found_file['owner'] != requestor_bare:
+            # only owner can manage files
+            log.warning(_("Only owner can manage files"))
+            raise WorkflowError(self._err(_("forbidden")))
+
+        session_data['found_file'] = found_file
+        session_data['namespace'] = namespace
+        return found_file
+
+    def _update_read_permission(self, access, allowed_jids):
+        if not allowed_jids:
+            if C.ACCESS_PERM_READ in access:
+                del access[C.ACCESS_PERM_READ]
+        elif allowed_jids == 'PUBLIC':
+            access[C.ACCESS_PERM_READ] = {
+                "type": C.ACCESS_TYPE_PUBLIC
+            }
+        else:
+            access[C.ACCESS_PERM_READ] = {
+                "type": C.ACCESS_TYPE_WHITELIST,
+                "jids": [j.full() for j in allowed_jids]
+            }
+
+    async def _update_dir(self, client, requestor, namespace, file_data, allowed_jids):
+        """Recursively update permission of a directory and all subdirectories
+
+        @param file_data(dict): metadata of the file
+        @param allowed_jids(list[jid.JID]): list of entities allowed to read the file
+        """
+        assert file_data['type'] == C.FILE_TYPE_DIRECTORY
+        files_data = await self.host.memory.get_files(
+            client, requestor, parent=file_data['id'], namespace=namespace)
+
+        for file_data in files_data:
+            if not file_data['access'].get(C.ACCESS_PERM_READ, {}):
+                log.debug("setting {perm} read permission for {name}".format(
+                    perm=allowed_jids, name=file_data['name']))
+                await self.host.memory.file_update(
+                    file_data['id'], 'access',
+                    partial(self._update_read_permission, allowed_jids=allowed_jids))
+            if file_data['type'] == C.FILE_TYPE_DIRECTORY:
+                await self._update_dir(client, requestor, namespace, file_data, 'PUBLIC')
+
+    async def _on_change_file(self, client, command_elt, session_data, action, node):
+        try:
+            x_elt = next(command_elt.elements(data_form.NS_X_DATA, "x"))
+            command_form = data_form.Form.fromElement(x_elt)
+        except StopIteration:
+            command_form = None
+
+        found_file = session_data.get('found_file')
+        requestor = session_data['requestor']
+        requestor_bare = requestor.userhostJID()
+
+        if command_form is None or len(command_form.fields) == 0:
+            # root request
+            return self._get_root_args()
+
+        elif found_file is None:
+            # file selected, we retrieve it and ask for permissions
+            try:
+                found_file = await self._get_file_data(client, session_data, command_form)
+            except WorkflowError as e:
+                return e.err_args
+
+            # management request
+            if found_file['type'] == C.FILE_TYPE_DIRECTORY:
+                instructions = D_("Please select permissions for this directory")
+            else:
+                instructions = D_("Please select permissions for this file")
+
+            form = data_form.Form("form", title="File Management",
+                                  instructions=[instructions],
+                                  formNamespace=NS_FILE_MANAGEMENT)
+            field = data_form.Field(
+                "text-multi", "read_allowed", required=False,
+                desc='list of jids allowed to read this file (beside yourself), or '
+                     '"PUBLIC" to let a public access'
+            )
+            read_access = found_file["access"].get(C.ACCESS_PERM_READ, {})
+            access_type = read_access.get('type', C.ACCESS_TYPE_WHITELIST)
+            if access_type == C.ACCESS_TYPE_PUBLIC:
+                field.values = ['PUBLIC']
+            else:
+                field.values = read_access.get('jids', [])
+            form.addField(field)
+            if found_file['type'] == C.FILE_TYPE_DIRECTORY:
+                field = data_form.Field(
+                    "boolean", "recursive", value=False, required=False,
+                    desc="Files under it will be made public to follow this dir "
+                         "permission (only if they don't have already a permission set)."
+                )
+                form.addField(field)
+
+            status = self._c.STATUS.EXECUTING
+            payload = form.toElement()
+            return (payload, status, None, None)
+
+        else:
+            # final phase, we'll do permission change here
+            try:
+                read_allowed = command_form.fields['read_allowed']
+            except KeyError:
+                self._c.ad_hoc_error(self._c.ERROR.BAD_PAYLOAD)
+
+            if read_allowed.value == 'PUBLIC':
+                allowed_jids = 'PUBLIC'
+            elif read_allowed.value.strip() == '':
+                allowed_jids = None
+            else:
+                try:
+                    allowed_jids = [jid.JID(v.strip()) for v in read_allowed.values
+                                    if v.strip()]
+                except RuntimeError as e:
+                    log.warning(_("Can't use read_allowed values: {reason}").format(
+                        reason=e))
+                    self._c.ad_hoc_error(self._c.ERROR.BAD_PAYLOAD)
+
+            if found_file['type'] == C.FILE_TYPE_FILE:
+                await self.host.memory.file_update(
+                    found_file['id'], 'access',
+                    partial(self._update_read_permission, allowed_jids=allowed_jids))
+            else:
+                try:
+                    recursive = command_form.fields['recursive']
+                except KeyError:
+                    self._c.ad_hoc_error(self._c.ERROR.BAD_PAYLOAD)
+                await self.host.memory.file_update(
+                    found_file['id'], 'access',
+                    partial(self._update_read_permission, allowed_jids=allowed_jids))
+                if recursive:
+                    # we set all file under the directory as public (if they haven't
+                    # already a permission set), so allowed entities of root directory
+                    # can read them.
+                    namespace = session_data['namespace']
+                    await self._update_dir(
+                        client, requestor_bare, namespace, found_file, 'PUBLIC')
+
+            # job done, we can end the session
+            status = self._c.STATUS.COMPLETED
+            payload = None
+            note = (self._c.NOTE.INFO, _("management session done"))
+            return (payload, status, None, note)
+
+    async def _on_delete_file(self, client, command_elt, session_data, action, node):
+        try:
+            x_elt = next(command_elt.elements(data_form.NS_X_DATA, "x"))
+            command_form = data_form.Form.fromElement(x_elt)
+        except StopIteration:
+            command_form = None
+
+        found_file = session_data.get('found_file')
+        requestor = session_data['requestor']
+        requestor_bare = requestor.userhostJID()
+
+        if command_form is None or len(command_form.fields) == 0:
+            # root request
+            return self._get_root_args()
+
+        elif found_file is None:
+            # file selected, we need confirmation before actually deleting
+            try:
+                found_file = await self._get_file_data(client, session_data, command_form)
+            except WorkflowError as e:
+                return e.err_args
+            if found_file['type'] == C.FILE_TYPE_DIRECTORY:
+                msg = D_("Are you sure to delete directory {name} and all files and "
+                         "directories under it?").format(name=found_file['name'])
+            else:
+                msg = D_("Are you sure to delete file {name}?"
+                    .format(name=found_file['name']))
+            form = data_form.Form("form", title="File Management",
+                                  instructions = [msg],
+                                  formNamespace=NS_FILE_MANAGEMENT)
+            field = data_form.Field(
+                "boolean", "confirm", value=False, required=True,
+                desc="check this box to confirm"
+            )
+            form.addField(field)
+            status = self._c.STATUS.EXECUTING
+            payload = form.toElement()
+            return (payload, status, None, None)
+
+        else:
+            # final phase, we'll do deletion here
+            try:
+                confirmed = C.bool(command_form.fields['confirm'].value)
+            except KeyError:
+                self._c.ad_hoc_error(self._c.ERROR.BAD_PAYLOAD)
+            if not confirmed:
+                note = None
+            else:
+                recursive = found_file['type'] == C.FILE_TYPE_DIRECTORY
+                await self.host.memory.file_delete(
+                    client, requestor_bare, found_file['id'], recursive)
+                note = (self._c.NOTE.INFO, _("file deleted"))
+            status = self._c.STATUS.COMPLETED
+            payload = None
+            return (payload, status, None, note)
+
+    def _update_thumbs(self, extra, thumbnails):
+        extra[C.KEY_THUMBNAILS] = thumbnails
+
+    async def _gen_thumbs(self, client, requestor, namespace, file_data):
+        """Recursively generate thumbnails
+
+        @param file_data(dict): metadata of the file
+        """
+        if file_data['type'] == C.FILE_TYPE_DIRECTORY:
+            sub_files_data = await self.host.memory.get_files(
+                client, requestor, parent=file_data['id'], namespace=namespace)
+            for sub_file_data in sub_files_data:
+                await self._gen_thumbs(client, requestor, namespace, sub_file_data)
+
+        elif file_data['type'] == C.FILE_TYPE_FILE:
+            media_type = file_data['media_type']
+            file_path = os.path.join(self.files_path, file_data['file_hash'])
+            if media_type == 'image':
+                thumbnails = []
+
+                for max_thumb_size in self._t.SIZES:
+                    try:
+                        thumb_size, thumb_id = await self._t.generate_thumbnail(
+                            file_path,
+                            max_thumb_size,
+                            #  we keep thumbnails for 6 months
+                            60 * 60 * 24 * 31 * 6,
+                        )
+                    except Exception as e:
+                        log.warning(_("Can't create thumbnail: {reason}")
+                            .format(reason=e))
+                        break
+                    thumbnails.append({"id": thumb_id, "size": thumb_size})
+
+                await self.host.memory.file_update(
+                    file_data['id'], 'extra',
+                    partial(self._update_thumbs, thumbnails=thumbnails))
+
+                log.info("thumbnails for [{file_name}] generated"
+                    .format(file_name=file_data['name']))
+
+        else:
+            log.warning("unmanaged file type: {type_}".format(type_=file_data['type']))
+
+    async def _on_gen_thumbnails(self, client, command_elt, session_data, action, node):
+        try:
+            x_elt = next(command_elt.elements(data_form.NS_X_DATA, "x"))
+            command_form = data_form.Form.fromElement(x_elt)
+        except StopIteration:
+            command_form = None
+
+        found_file = session_data.get('found_file')
+        requestor = session_data['requestor']
+
+        if command_form is None or len(command_form.fields) == 0:
+            # root request
+            return self._get_root_args()
+
+        elif found_file is None:
+            # file selected, we retrieve it and ask for permissions
+            try:
+                found_file = await self._get_file_data(client, session_data, command_form)
+            except WorkflowError as e:
+                return e.err_args
+
+            log.info("Generating thumbnails as requested")
+            await self._gen_thumbs(client, requestor, found_file['namespace'], found_file)
+
+            # job done, we can end the session
+            status = self._c.STATUS.COMPLETED
+            payload = None
+            note = (self._c.NOTE.INFO, _("thumbnails generated"))
+            return (payload, status, None, note)
+
+    async def _on_quota(self, client, command_elt, session_data, action, node):
+        requestor = session_data['requestor']
+        quota = self.host.plugins["file_sharing"].get_quota(client, requestor)
+        try:
+            size_used = await self.host.memory.file_get_used_space(client, requestor)
+        except exceptions.PermissionError:
+            raise WorkflowError(self._err(_("forbidden")))
+        status = self._c.STATUS.COMPLETED
+        form = data_form.Form("result")
+        form.makeFields({"quota": quota, "user": size_used})
+        payload = form.toElement()
+        note = (
+            self._c.NOTE.INFO,
+            _("You are currently using {size_used} on {size_quota}").format(
+                size_used = utils.get_human_size(size_used),
+                size_quota = (
+                    _("unlimited quota") if quota is None
+                    else utils.get_human_size(quota)
+                )
+            )
+        )
+        return (payload, status, None, note)

libervia/backend/plugins/plugin_dbg_manhole.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+
+# SAT plugin for debugging, using a manhole
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from libervia.backend.core.i18n import _
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.log import getLogger
+from twisted.conch.insults import insults
+from twisted.conch.telnet import TelnetTransport, TelnetBootstrapProtocol
+from twisted.internet import reactor, protocol, defer
+from twisted.words.protocols.jabber import jid
+from twisted.conch.manhole import ColoredManhole
+
+log = getLogger(__name__)
+
+PLUGIN_INFO = {
+    C.PI_NAME: "Manhole debug plugin",
+    C.PI_IMPORT_NAME: "manhole",
+    C.PI_TYPE: "DEBUG",
+    C.PI_PROTOCOLS: [],
+    C.PI_DEPENDENCIES: [],
+    C.PI_MAIN: "Manhole",
+    C.PI_HANDLER: "no",
+    C.PI_DESCRIPTION: _("""Debug plugin to have a telnet server"""),
+}
+
+
+
+class Manhole(object):
+
+    def __init__(self, host):
+        self.host = host
+        port = int(host.memory.config_get(None, "manhole_debug_dangerous_port_int", 0))
+        if port:
+            self.start_manhole(port)
+
+    def start_manhole(self, port):
+        log.warning(_("/!\\ Manhole debug server activated, be sure to not use it in "
+                      "production, this is dangerous /!\\"))
+        log.info(_("You can connect to manhole server using telnet on port {port}")
+            .format(port=port))
+        f = protocol.ServerFactory()
+        namespace = {
+            "host": self.host,
+            "C": C,
+            "jid": jid,
+            "d": defer.ensureDeferred,
+        }
+        f.protocol = lambda: TelnetTransport(TelnetBootstrapProtocol,
+                                             insults.ServerProtocol,
+                                             ColoredManhole,
+                                             namespace=namespace,
+                                             )
+        reactor.listenTCP(port, f)

libervia/backend/plugins/plugin_exp_command_export.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+
+
+# SAT plugin to export commands (experimental)
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from libervia.backend.core.i18n import _
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.log import getLogger
+
+log = getLogger(__name__)
+from twisted.words.protocols.jabber import jid
+from twisted.internet import reactor, protocol
+
+from libervia.backend.tools import trigger
+from libervia.backend.tools.utils import clean_ustr
+
+PLUGIN_INFO = {
+    C.PI_NAME: "Command export plugin",
+    C.PI_IMPORT_NAME: "EXP-COMMANS-EXPORT",
+    C.PI_TYPE: "EXP",
+    C.PI_PROTOCOLS: [],
+    C.PI_DEPENDENCIES: [],
+    C.PI_MAIN: "CommandExport",
+    C.PI_HANDLER: "no",
+    C.PI_DESCRIPTION: _("""Implementation of command export"""),
+}
+
+
+class ExportCommandProtocol(protocol.ProcessProtocol):
+    """ Try to register an account with prosody """
+
+    def __init__(self, parent, client, target, options):
+        self.parent = parent
+        self.target = target
+        self.options = options
+        self.client = client
+
+    def _clean(self, data):
+        if not data:
+            log.error("data should not be empty !")
+            return ""
+        decoded = data.decode("utf-8", "ignore")[: -1 if data[-1] == "\n" else None]
+        return clean_ustr(decoded)
+
+    def connectionMade(self):
+        log.info("connectionMade :)")
+
+    def outReceived(self, data):
+        self.client.sendMessage(self.target, {"": self._clean(data)}, no_trigger=True)
+
+    def errReceived(self, data):
+        self.client.sendMessage(self.target, {"": self._clean(data)}, no_trigger=True)
+
+    def processEnded(self, reason):
+        log.info("process finished: %d" % (reason.value.exitCode,))
+        self.parent.removeProcess(self.target, self)
+
+    def write(self, message):
+        self.transport.write(message.encode("utf-8"))
+
+    def bool_option(self, key):
+        """ Get boolean value from options
+        @param key: name of the option
+        @return: True if key exists and set to "true" (case insensitive),
+                 False in all other cases """
+        value = self.options.get(key, "")
+        return value.lower() == "true"
+
+
+class CommandExport(object):
+    """Command export plugin: export a command to an entity"""
+
+    # XXX: This plugin can be potentially dangerous if we don't trust entities linked
+    #      this is specially true if we have other triggers.
+    # FIXME: spawned should be a client attribute, not a class one
+
+    def __init__(self, host):
+        log.info(_("Plugin command export initialization"))
+        self.host = host
+        self.spawned = {}  # key = entity
+        host.trigger.add("message_received", self.message_received_trigger, priority=10000)
+        host.bridge.add_method(
+            "command_export",
+            ".plugin",
+            in_sign="sasasa{ss}s",
+            out_sign="",
+            method=self._export_command,
+        )
+
+    def removeProcess(self, entity, process):
+        """ Called when the process is finished
+        @param entity: jid.JID attached to the process
+        @param process: process to remove"""
+        try:
+            processes_set = self.spawned[(entity, process.client.profile)]
+            processes_set.discard(process)
+            if not processes_set:
+                del (self.spawned[(entity, process.client.profile)])
+        except ValueError:
+            pass
+
+    def message_received_trigger(self, client, message_elt, post_treat):
+        """ Check if source is linked and repeat message, else do nothing  """
+        from_jid = jid.JID(message_elt["from"])
+        spawned_key = (from_jid.userhostJID(), client.profile)
+
+        if spawned_key in self.spawned:
+            try:
+                body = next(message_elt.elements(C.NS_CLIENT, "body"))
+            except StopIteration:
+                # do not block message without body (chat state notification...)
+                return True
+
+            mess_data = str(body) + "\n"
+            processes_set = self.spawned[spawned_key]
+            _continue = False
+            exclusive = False
+            for process in processes_set:
+                process.write(mess_data)
+                _continue &= process.bool_option("continue")
+                exclusive |= process.bool_option("exclusive")
+            if exclusive:
+                raise trigger.SkipOtherTriggers
+            return _continue
+
+        return True
+
+    def _export_command(self, command, args, targets, options, profile_key):
+        """ Export a commands to authorised targets
+        @param command: full path of the command to execute
+        @param args: list of arguments, with command name as first one
+        @param targets: list of allowed entities
+        @param options: export options, a dict which can have the following keys ("true" to set booleans):
+                        - exclusive: if set, skip all other triggers
+                        - loop: if set, restart the command once terminated #TODO
+                        - pty: if set, launch in a pseudo terminal
+                        - continue: continue normal message_received handling
+        """
+        client = self.host.get_client(profile_key)
+        for target in targets:
+            try:
+                _jid = jid.JID(target)
+                if not _jid.user or not _jid.host:
+                    raise jid.InvalidFormat
+                _jid = _jid.userhostJID()
+            except (RuntimeError, jid.InvalidFormat, AttributeError):
+                log.info("invalid target ignored: %s" % (target,))
+                continue
+            process_prot = ExportCommandProtocol(self, client, _jid, options)
+            self.spawned.setdefault((_jid, client.profile), set()).add(process_prot)
+            reactor.spawnProcess(
+                process_prot, command, args, usePTY=process_prot.bool_option("pty")
+            )

libervia/backend/plugins/plugin_exp_invitation.py

@@ -0,0 +1,350 @@
+#!/usr/bin/env python3
+
+# SàT plugin to manage invitations
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import Optional
+from zope.interface import implementer
+from twisted.internet import defer
+from twisted.words.protocols.jabber import jid
+from twisted.words.protocols.jabber.xmlstream import XMPPHandler
+from wokkel import disco, iwokkel
+from libervia.backend.core.i18n import _
+from libervia.backend.core import exceptions
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.log import getLogger
+from libervia.backend.core.xmpp import SatXMPPEntity
+from libervia.backend.tools import utils
+
+log = getLogger(__name__)
+
+
+PLUGIN_INFO = {
+    C.PI_NAME: "Invitation",
+    C.PI_IMPORT_NAME: "INVITATION",
+    C.PI_TYPE: "EXP",
+    C.PI_PROTOCOLS: [],
+    C.PI_DEPENDENCIES: ["XEP-0060", "XEP-0329", "XEP-0334", "LIST_INTEREST"],
+    C.PI_RECOMMENDATIONS: ["EMAIL_INVITATION"],
+    C.PI_MAIN: "Invitation",
+    C.PI_HANDLER: "yes",
+    C.PI_DESCRIPTION: _("Experimental handling of invitations"),
+}
+
+NS_INVITATION = "https://salut-a-toi/protocol/invitation:0"
+INVITATION = '/message/invitation[@xmlns="{ns_invit}"]'.format(
+    ns_invit=NS_INVITATION
+)
+NS_INVITATION_LIST = NS_INVITATION + "#list"
+
+
+class Invitation(object):
+
+    def __init__(self, host):
+        log.info(_("Invitation plugin initialization"))
+        self.host = host
+        self._p = self.host.plugins["XEP-0060"]
+        self._h = self.host.plugins["XEP-0334"]
+        # map from namespace of the invitation to callback handling it
+        self._ns_cb = {}
+
+    def get_handler(self, client):
+        return PubsubInvitationHandler(self)
+
+    def register_namespace(self, namespace, callback):
+        """Register a callback for a namespace
+
+        @param namespace(unicode): namespace handled
+        @param callback(callbable): method handling the invitation
+            For pubsub invitation, it will be called with following arguments:
+                - client
+                - name(unicode, None): name of the event
+                - extra(dict): extra data
+                - service(jid.JID): pubsub service jid
+                - node(unicode): pubsub node
+                - item_id(unicode, None): pubsub item id
+                - item_elt(domish.Element): item of the invitation
+            For file sharing invitation, it will be called with following arguments:
+                - client
+                - name(unicode, None): name of the repository
+                - extra(dict): extra data
+                - service(jid.JID): service jid of the file repository
+                - repos_type(unicode): type of the repository, can be:
+                    - files: generic file sharing
+                    - photos: photos album
+                - namespace(unicode, None): namespace of the repository
+                - path(unicode, None): path of the repository
+        @raise exceptions.ConflictError: this namespace is already registered
+        """
+        if namespace in self._ns_cb:
+            raise exceptions.ConflictError(
+                "invitation namespace {namespace} is already register with {callback}"
+                .format(namespace=namespace, callback=self._ns_cb[namespace]))
+        self._ns_cb[namespace] = callback
+
+    def _generate_base_invitation(self, client, invitee_jid, name, extra):
+        """Generate common mess_data end invitation_elt
+
+        @param invitee_jid(jid.JID): entitee to send invitation to
+        @param name(unicode, None): name of the shared repository
+        @param extra(dict, None): extra data, where key can be:
+            - thumb_url: URL of a thumbnail
+        @return (tuple[dict, domish.Element): mess_data and invitation_elt
+        """
+        mess_data = {
+            "from": client.jid,
+            "to": invitee_jid,
+            "uid": "",
+            "message": {},
+            "type": C.MESS_TYPE_CHAT,
+            "subject": {},
+            "extra": {},
+        }
+        client.generate_message_xml(mess_data)
+        self._h.add_hint_elements(mess_data["xml"], [self._h.HINT_STORE])
+        invitation_elt = mess_data["xml"].addElement("invitation", NS_INVITATION)
+        if name is not None:
+            invitation_elt["name"] = name
+        thumb_url = extra.get('thumb_url')
+        if thumb_url:
+            if not thumb_url.startswith('http'):
+                log.warning(
+                    "only http URLs are allowed for thumbnails, got {url}, ignoring"
+                    .format(url=thumb_url))
+            else:
+                invitation_elt['thumb_url'] = thumb_url
+        return mess_data, invitation_elt
+
+    def send_pubsub_invitation(
+        self,
+        client: SatXMPPEntity,
+        invitee_jid: jid.JID,
+        service: jid.JID,
+        node: str,
+        item_id: Optional[str],
+        name: Optional[str],
+        extra: Optional[dict]
+    ) -> None:
+        """Send an pubsub invitation in a <message> stanza
+
+        @param invitee_jid: entitee to send invitation to
+        @param service: pubsub service
+        @param node: pubsub node
+        @param item_id: pubsub id
+            None when the invitation is for a whole node
+        @param name: see [_generate_base_invitation]
+        @param extra: see [_generate_base_invitation]
+        """
+        if extra is None:
+            extra = {}
+        mess_data, invitation_elt = self._generate_base_invitation(
+            client, invitee_jid, name, extra)
+        pubsub_elt = invitation_elt.addElement("pubsub")
+        pubsub_elt["service"] = service.full()
+        pubsub_elt["node"] = node
+        if item_id is None:
+            try:
+                namespace = extra.pop("namespace")
+            except KeyError:
+                raise exceptions.DataError('"namespace" key is missing in "extra" data')
+            node_data_elt = pubsub_elt.addElement("node_data")
+            node_data_elt["namespace"] = namespace
+            try:
+                node_data_elt.addChild(extra["element"])
+            except KeyError:
+                pass
+        else:
+            pubsub_elt["item"] = item_id
+        if "element" in extra:
+            invitation_elt.addChild(extra.pop("element"))
+        client.send(mess_data["xml"])
+
+    async def send_file_sharing_invitation(
+        self, client, invitee_jid, service, repos_type=None, namespace=None, path=None,
+        name=None, extra=None
+    ):
+        """Send a file sharing invitation in a <message> stanza
+
+        @param invitee_jid(jid.JID): entitee to send invitation to
+        @param service(jid.JID): file sharing service
+        @param repos_type(unicode, None): type of files repository, can be:
+            - None, "files": files sharing
+            - "photos": photos album
+        @param namespace(unicode, None): namespace of the shared repository
+        @param path(unicode, None): path of the shared repository
+        @param name(unicode, None): see [_generate_base_invitation]
+        @param extra(dict, None): see [_generate_base_invitation]
+        """
+        if extra is None:
+            extra = {}
+        li_plg = self.host.plugins["LIST_INTEREST"]
+        li_plg.normalise_file_sharing_service(client, service)
+
+        # FIXME: not the best place to adapt permission, but it's necessary to check them
+        #   for UX
+        try:
+            await self.host.plugins['XEP-0329'].affiliationsSet(
+                client, service, namespace, path, {invitee_jid: "member"}
+            )
+        except Exception as e:
+            log.warning(f"Can't set affiliation: {e}")
+
+        if "thumb_url" not in extra:
+            # we have no thumbnail, we check in our own list of interests if there is one
+            try:
+                item_id = li_plg.get_file_sharing_id(service, namespace, path)
+                own_interest = await li_plg.get(client, item_id)
+            except exceptions.NotFound:
+                log.debug(
+                    "no thumbnail found for file sharing interest at "
+                    f"[{service}/{namespace}]{path}"
+                )
+            else:
+                try:
+                    extra['thumb_url'] = own_interest['thumb_url']
+                except KeyError:
+                    pass
+
+        mess_data, invitation_elt = self._generate_base_invitation(
+            client, invitee_jid, name, extra)
+        file_sharing_elt = invitation_elt.addElement("file_sharing")
+        file_sharing_elt["service"] = service.full()
+        if repos_type is not None:
+            if repos_type not in ("files", "photos"):
+                msg = "unknown repository type: {repos_type}".format(
+                    repos_type=repos_type)
+                log.warning(msg)
+                raise exceptions.DateError(msg)
+            file_sharing_elt["type"] = repos_type
+        if namespace is not None:
+            file_sharing_elt["namespace"] = namespace
+        if path is not None:
+            file_sharing_elt["path"] = path
+        client.send(mess_data["xml"])
+
+    async def _parse_pubsub_elt(self, client, pubsub_elt):
+        try:
+            service = jid.JID(pubsub_elt["service"])
+            node = pubsub_elt["node"]
+        except (RuntimeError, KeyError):
+            raise exceptions.DataError("Bad invitation, ignoring")
+
+        item_id = pubsub_elt.getAttribute("item")
+
+        if item_id is not None:
+            try:
+                items, metadata = await self._p.get_items(
+                    client, service, node, item_ids=[item_id]
+                )
+            except Exception as e:
+                log.warning(_("Can't get item linked with invitation: {reason}").format(
+                            reason=e))
+            try:
+                item_elt = items[0]
+            except IndexError:
+                log.warning(_("Invitation was linking to a non existing item"))
+                raise exceptions.DataError
+
+            try:
+                namespace = item_elt.firstChildElement().uri
+            except Exception as e:
+                log.warning(_("Can't retrieve namespace of invitation: {reason}").format(
+                    reason = e))
+                raise exceptions.DataError
+
+            args = [service, node, item_id, item_elt]
+        else:
+            try:
+                node_data_elt = next(pubsub_elt.elements(NS_INVITATION, "node_data"))
+            except StopIteration:
+                raise exceptions.DataError("Bad invitation, ignoring")
+            namespace = node_data_elt['namespace']
+            args = [service, node, None, node_data_elt]
+
+        return namespace, args
+
+    async def _parse_file_sharing_elt(self, client, file_sharing_elt):
+        try:
+            service = jid.JID(file_sharing_elt["service"])
+        except (RuntimeError, KeyError):
+            log.warning(_("Bad invitation, ignoring"))
+            raise exceptions.DataError
+        repos_type = file_sharing_elt.getAttribute("type", "files")
+        sharing_ns = file_sharing_elt.getAttribute("namespace")
+        path = file_sharing_elt.getAttribute("path")
+        args = [service, repos_type, sharing_ns, path]
+        ns_fis = self.host.get_namespace("fis")
+        return ns_fis, args
+
+    async def on_invitation(self, message_elt, client):
+        log.debug("invitation received [{profile}]".format(profile=client.profile))
+        invitation_elt = message_elt.invitation
+
+        name = invitation_elt.getAttribute("name")
+        extra = {}
+        if invitation_elt.hasAttribute("thumb_url"):
+            extra['thumb_url'] = invitation_elt['thumb_url']
+
+        for elt in invitation_elt.elements():
+            if elt.uri != NS_INVITATION:
+                log.warning("unexpected element: {xml}".format(xml=elt.toXml()))
+                continue
+            if elt.name == "pubsub":
+                method = self._parse_pubsub_elt
+            elif elt.name == "file_sharing":
+                method = self._parse_file_sharing_elt
+            else:
+                log.warning("not implemented invitation element: {xml}".format(
+                    xml = elt.toXml()))
+                continue
+            try:
+                namespace, args = await method(client, elt)
+            except exceptions.DataError:
+                log.warning("Can't parse invitation element: {xml}".format(
+                            xml = elt.toXml()))
+                continue
+
+            try:
+                cb = self._ns_cb[namespace]
+            except KeyError:
+                log.warning(_(
+                    'No handler for namespace "{namespace}", invitation ignored')
+                    .format(namespace=namespace))
+            else:
+                await utils.as_deferred(cb, client, namespace, name, extra, *args)
+
+
+@implementer(iwokkel.IDisco)
+class PubsubInvitationHandler(XMPPHandler):
+
+    def __init__(self, plugin_parent):
+        self.plugin_parent = plugin_parent
+
+    def connectionInitialized(self):
+        self.xmlstream.addObserver(
+            INVITATION,
+            lambda message_elt: defer.ensureDeferred(
+                self.plugin_parent.on_invitation(message_elt, client=self.parent)
+            ),
+        )
+
+    def getDiscoInfo(self, requestor, target, nodeIdentifier=""):
+        return [
+            disco.DiscoFeature(NS_INVITATION),
+        ]
+
+    def getDiscoItems(self, requestor, target, nodeIdentifier=""):
+        return []

libervia/backend/plugins/plugin_exp_invitation_file.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+# SàT plugin to send invitations for file sharing
+# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from libervia.backend.core.i18n import _
+from libervia.backend.core.constants import Const as C
+from libervia.backend.core.log import getLogger
+from libervia.backend.core.xmpp import SatXMPPEntity
+from libervia.backend.tools.common import data_format
+from twisted.internet import defer
+from twisted.words.protocols.jabber import jid
+
+log = getLogger(__name__)
+
+
+PLUGIN_INFO = {
+    C.PI_NAME: "File Sharing Invitation",
+    C.PI_IMPORT_NAME: "FILE_SHARING_INVITATION",
+    C.PI_TYPE: "EXP",
+    C.PI_PROTOCOLS: [],
+    C.PI_DEPENDENCIES: ["XEP-0329", "INVITATION"],
+    C.PI_RECOMMENDATIONS: [],
+    C.PI_MAIN: "FileSharingInvitation",
+    C.PI_HANDLER: "no",
+    C.PI_DESCRIPTION: _("Experimental handling of invitations for file sharing"),
+}
+
+
+class FileSharingInvitation:
+
+    def __init__(self, host):
+        log.info(_("File Sharing Invitation plugin initialization"))
+        self.host = host
+        ns_fis = host.get_namespace("fis")
+        host.plugins["INVITATION"].register_namespace(ns_fis, self.on_invitation)
+        host.bridge.add_method(
+            "fis_invite",
+            ".plugin",
+            in_sign="ssssssss",
+            out_sign="",
+            method=self._send_file_sharing_invitation,
+            async_=True
+        )
+
+    def _send_file_sharing_invitation(
+            self, invitee_jid_s, service_s, repos_type=None, namespace=None, path=None,
+            name=None, extra_s='', profile_key=C.PROF_KEY_NONE):
+        client = self.host.get_client(profile_key)
+        invitee_jid = jid.JID(invitee_jid_s)
+        service = jid.JID(service_s)
+        extra = data_format.deserialise(extra_s)
+        return defer.ensureDeferred(
+            self.host.plugins["INVITATION"].send_file_sharing_invitation(
+                client, invitee_jid, service, repos_type=repos_type or None,
+                namespace=namespace or None, path=path or None, name=name or None,
+                extra=extra)
+        )
+
+    def on_invitation(
+        self,
+        client: SatXMPPEntity,
+        namespace: str,
+        name: str,
+        extra: dict,
+        service: jid.JID,
+        repos_type: str,
+        sharing_ns: str,
+        path: str
+    ):
+        if repos_type == "files":
+            type_human = _("file sharing")
+        elif repos_type == "photos":
+            type_human = _("photo album")
+        else:
+            log.warning("Unknown repository type: {repos_type}".format(
+                repos_type=repos_type))
+            repos_type = "file"
+            type_human = _("file sharing")
+        log.info(_(
+            '{profile} has received an invitation for a files repository ({type_human}) '
+            'with namespace {sharing_ns!r} at path [{path}]').format(
+            profile=client.profile, type_human=type_human, sharing_ns=sharing_ns,
+                path=path)
+            )
+        return defer.ensureDeferred(
+            self.host.plugins['LIST_INTEREST'].register_file_sharing(
+                client, service, repos_type, sharing_ns, path, name, extra
+            )
+        )