merge bookmark @

.hgignore

@@ -7,6 +7,7 @@ tags
 twistd.log
 twistd.pid
 bridge_constructor/generated
+tests/e2e/report*
 _trial_temp/
 sat.egg-info
 *.un~

CHANGELOG

@@ -100,7 +100,7 @@ v 0.8.0 « La Cecília » (NOT RELEASED YET):
         - bulma theme becomes the default one for default site
         - "tickets" have been renamed to more generic term "lists"
         - (lists) common lists can be created from templates
-        - (photos) album can now be created or deleted from Libervia
+        - (photos) album can now be created or deleted from Libervia Web
         - (photos/album) photos or videos can now be uploaded from Libervia
         - (photos/album) a mobile friendly slideshow can be run
         - (photos/album) affiliations can be modified with new invitations manager

docker/backend-dev-demo/Dockerfile

@@ -16,7 +16,7 @@ USER libervia
 
 RUN ./entrypoint.sh \
     # we create the file sharing component which will autoconnect when backend is started
-    jp profile create file_sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file_sharing -A && \
+    jp profile create file-sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file-sharing -A && \
     libervia-backend stop
 
 RUN ./entrypoint.sh \

docker/backend-dev-demo/libervia.conf

@@ -4,7 +4,7 @@ email_server = libervia-backend.test
 email_port = 8025
 email_admins_list = admin@server1.test
 
-[component file_sharing]
+[component file-sharing]
 http_upload_public_facing_url = http://localhost:7777
 http_upload_connection_type = http
 http_upload_port = 7777

docker/backend-dev-e2e/Dockerfile

@@ -1,5 +1,5 @@
-ARG REVISION=dev
+ARG REVISION
-FROM libervia/backend:${REVISION}
+FROM libervia/backend:${REVISION:-dev}
 
 LABEL maintainer="Goffi <tmp_dockerfiles@goffi.org>"
 
@@ -58,7 +58,7 @@ RUN \
 
 RUN ./entrypoint.sh \
     # we create the file sharing component which will autoconnect when backend is started
-    li profile create file_sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file_sharing -A && \
+    li profile create file-sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file-sharing -A && \
     libervia-backend stop
 
 ENV LIBERVIA_TEST_REPORT_DIR=/reports

docker/backend-dev-e2e/libervia.conf

@@ -4,7 +4,7 @@ email_server = libervia-backend.test
 email_port = 8025
 email_admins_list = admin@server1.test
 
-[component file_sharing]
+[component file-sharing]
 tls_certificate = /usr/share/libervia/certificates/server1.test.pem
 tls_private_key = /usr/share/libervia/certificates/server1.test-key.pem
 http_upload_public_facing_url = https://libervia-backend.test:8888

docker/backend-dev/Dockerfile

@@ -36,12 +36,12 @@ RUN python3 -m venv libervia_env && libervia_env/bin/pip install -U pip wheel &&
     # local repos without it them cause troubles
     hg clone https://repos.goffi.org/urwid-satext && ~/libervia_env/bin/pip install -e urwid-satext && \
     mv urwid-satext/urwid_satext.egg-info ~/libervia_env/lib/python3.*/site-packages && \
-    hg clone https://repos.goffi.org/sat_tmp -u "${REVISION:-tip}" && ~/libervia_env/bin/pip install -e sat_tmp && \
+    hg clone https://repos.goffi.org/sat_tmp -u "${REVISION:-@}" && ~/libervia_env/bin/pip install -e sat_tmp && \
     mv sat_tmp/sat_tmp.egg-info ~/libervia_env/lib/python3.*/site-packages && \
     hg clone https://repos.goffi.org/libervia-templates && ~/libervia_env/bin/pip install -e libervia-templates && \
     mv libervia-templates/libervia_templates.egg-info ~/libervia_env/lib/python3.*/site-packages && \
     hg clone https://repos.goffi.org/libervia-media && \
-    hg clone https://repos.goffi.org/libervia-backend -u "${REVISION:-tip}" && ~/libervia_env/bin/pip install -e 'libervia-backend[SVG]' && \
+    hg clone https://repos.goffi.org/libervia-backend -u "${REVISION:-@}" && ~/libervia_env/bin/pip install -e 'libervia-backend[SVG]' && \
     mv libervia-backend/libervia_backend.egg-info ~/libervia_env/lib/python3.*/site-packages && \
     mkdir -p /home/libervia/.local/share/libervia
 

docker/docker-compose-e2e.yml

@@ -7,7 +7,6 @@ services:
     depends_on:
       # we need to depend on backend to get IP address of the container for conf
       - backend
-    tmpfs: /var/lib/prosody
     tmpfs: /var/log/prosody
     networks:
       default:

docker/libervia-web-dev-demo/Dockerfile

@@ -1,5 +1,5 @@
-ARG REVISION=dev
+ARG REVISION
-FROM libervia/web:${REVISION}
+FROM libervia/web:${REVISION:-dev}
 
 LABEL maintainer="Goffi <tmp_dockerfiles@goffi.org>"
 

docker/libervia-web-dev-e2e/Dockerfile

@@ -1,5 +1,5 @@
-ARG REVISION=dev
+ARG REVISION
-FROM libervia/web:${REVISION}
+FROM libervia/web:${REVISION:-dev}
 
 LABEL maintainer="Goffi <tmp_dockerfiles@goffi.org>"
 

docker/libervia-web-dev/Dockerfile

@@ -11,7 +11,7 @@ USER root
 RUN apt-get install -y --no-install-recommends yarnpkg
 WORKDIR /home/libervia
 USER libervia
-RUN cd /src && hg clone https://repos.goffi.org/libervia-web -u "${REVISION:-tip}" && \
+RUN cd /src && hg clone https://repos.goffi.org/libervia-web -u "${REVISION:-@}" && \
     ~/libervia_env/bin/pip install -e libervia-web && \
     mv libervia-web/libervia_web.egg-info ~/libervia_env/lib/python3.*/site-packages
 

docker/prosody-e2e/Dockerfile

@@ -16,7 +16,3 @@ COPY --chown=root:prosody prosody.cfg.lua /etc/prosody/prosody.cfg.lua
 COPY --chown=root:tls-cert certificates/server1.test/cert.pem /usr/share/libervia/certificates/server1.test.pem
 COPY --chown=root:tls-cert certificates/server1.test/key.pem /usr/share/libervia/certificates/server1.test-key.pem
 
-# we add exec to handle properly signals, this is missing upstream
-# FIXME: to be removed when new images are generated with
-#        https://github.com/prosody/prosody-docker/pull/65
-RUN sed -i "s/^runuser -u prosody/exec \0/" /entrypoint.sh

docker/prosody-e2e/entrypoint.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-set -e
-
-usermod -u "$(stat -c %u /var/lib/prosody/.)" prosody
-
-if [[ "$1" != "prosody" ]]; then
-    exec prosodyctl "$@"
-    exit 0;
-fi
-
-if [ "$LOCAL" -a  "$PASSWORD" -a "$DOMAIN" ] ; then
-    prosodyctl register "$LOCAL" "$DOMAIN" "$PASSWORD"
-fi
-
-exec runuser -u prosody -- "$@"

docker/web-demo.yml

@@ -6,8 +6,8 @@ services:
     depends_on:
       # we need to depend on backend to get IP address of the container for conf
       - backend
-    tmpfs: /var/lib/prosody
+    tmpfs:
-    tmpfs: /var/log/prosody
+      - /var/log/prosody
     networks:
       default:
         aliases:

requirements.txt

@@ -1,22 +1,18 @@
 attrs==21.2.0
 Automat==20.2.0
 Babel==2.9.1
-cairocffi==1.2.0
+certifi==2021.10.8
-CairoSVG==2.5.2
+cffi==1.15.0
-certifi==2021.5.30
+charset-normalizer==2.0.7
-cffi==1.14.5
-chardet==4.0.0
 constantly==15.1.0
-cryptography==3.4.7
+cryptography==3.4.8
-cssselect2==0.4.1
+dbus-python==1.2.18
-dbus-python==1.2.16
-defusedxml==0.7.1
 DoubleRatchet==0.7.0
 html2text==2020.1.16
 hyperlink==21.0.0
-idna==2.10
+idna==3.3
 incremental==21.3.0
-Jinja2==3.0.1
+Jinja2==3.0.2
 langid==1.1.6
 lxml==4.6.3
 Markdown==3.3.4
@@ -24,38 +20,36 @@ MarkupSafe==2.0.1
 miniupnpc==2.0.2
 mutagen==1.45.1
 netifaces==0.11.0
-numpy==1.20.3
+numpy==1.21.3
 OMEMO==0.12.0
 omemo-backend-signal==0.2.6
-Pillow==8.2.0
+Pillow==8.4.0
-progressbar2==3.53.1
+progressbar2==3.53.3
-protobuf==3.17.3
+protobuf==3.19.1
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pycairo==1.20.1
 pycparser==2.20
 pycrypto==2.6.1
-Pygments==2.9.0
+Pygments==2.10.0
-PyGObject==3.40.1
+PyGObject==3.40.0
 PyNaCl==1.4.0
 pyOpenSSL==20.0.1
-python-dateutil==2.8.1
+python-dateutil==2.8.2
 python-potr==1.0.2
 python-utils==2.5.6
-pytz==2021.1
+pytz==2021.3
 pyxdg==0.27
 PyYAML==5.4.1
-requests==2.25.1
+requests==2.26.0
 service-identity==21.1.0
-setuptools-scm==6.0.1
 shortuuid==1.0.1
 six==1.16.0
-tinycss2==1.1.0
 treq==21.5.0
 Twisted==21.2.0
-urllib3==1.26.5
+typing-extensions==3.10.0.2
+urllib3==1.26.7
 urwid==2.1.2
-webencodings==0.5.1
 wokkel==18.0.0
 X3DH==0.5.9
 XEdDSA==0.4.7

sat/core/xmpp.py

@@ -303,7 +303,11 @@ class SatXMPPEntity(core_types.SatXMPPEntity):
         self.streamInitialized()
 
     def _finish_connection(self, __):
-        self.conn_deferred.callback(None)
+        if self.conn_deferred.called:
+            # can happen in case of forced disconnection by server
+            log.debug(f"{self} has already been connected")
+        else:
+            self.conn_deferred.callback(None)
 
     def streamInitialized(self):
         """Called after _authd"""

sat/plugins/plugin_comp_file_sharing.py

@@ -545,7 +545,7 @@ class FileSharing:
         self._f.openFileWrite(
             client, file_tmp_path, transfer_data, file_data, stream_object
         )
-        return False, defer.succeed(True)
+        return False, True
 
     async def _retrieveFiles(
         self, client, session, content_data, content_name, file_data, file_elt

sat/plugins/plugin_exp_events.py

@@ -428,6 +428,7 @@ class Events(object):
                 )
             else:
                 timestamp, data = self._parseEventElt(event_elt)
+                data["interest_id"] = item["id"]
                 events.append((timestamp, data))
         defer.returnValue(events)
 

sat/plugins/plugin_misc_email_invitation.py

@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import shortuuid
+from typing import Optional
 from twisted.internet import defer
 from twisted.words.protocols.jabber import jid
 from twisted.words.protocols.jabber import error
@@ -131,6 +132,97 @@ class InvitationsPlugin(object):
                 kwargs[key] = str(value)
         return defer.ensureDeferred(self.create(**kwargs))
 
+    async def getExistingInvitation(self, email: Optional[str]) -> Optional[dict]:
+        """Retrieve existing invitation with given email
+
+        @param email: check if any invitation exist with this email
+        @return: first found invitation, or None if nothing found
+        """
+        # FIXME: This method is highly inefficient, it get all invitations and check them
+        # one by one, this is just a temporary way to avoid creating creating new accounts
+        # for an existing email. A better way will be available with Libervia 0.9.
+        # TODO: use a better way to check existing invitations
+
+        if email is None:
+            return None
+        all_invitations = await self.invitations.all()
+        for id_, invitation in all_invitations.items():
+            if invitation.get("email") == email:
+                invitation[KEY_ID] = id_
+                return invitation
+
+    async def _createAccountAndProfile(
+        self,
+        id_: str,
+        kwargs: dict,
+        extra: dict
+    ) -> None:
+        """Create XMPP account and Libervia profile for guest"""
+        ## XMPP account creation
+        password = kwargs.pop('password', None)
+        if password is None:
+           password = utils.generatePassword()
+        assert password
+        # XXX: password is here saved in clear in database
+        #      it is needed for invitation as the same password is used for profile
+        #      and SàT need to be able to automatically open the profile with the uuid
+        # FIXME: we could add an extra encryption key which would be used with the
+        #        uuid when the invitee is connecting (e.g. with URL). This key would
+        #        not be saved and could be used to encrypt profile password.
+        extra[KEY_PASSWORD] = password
+
+        jid_ = kwargs.pop('jid_', None)
+        if not jid_:
+            domain = self.host.memory.getConfig(None, 'xmpp_domain')
+            if not domain:
+                # TODO: fallback to profile's domain
+                raise ValueError(_("You need to specify xmpp_domain in sat.conf"))
+            jid_ = "invitation-{uuid}@{domain}".format(uuid=shortuuid.uuid(),
+                                                        domain=domain)
+        jid_ = jid.JID(jid_)
+        extra[KEY_JID] = jid_.full()
+
+        if jid_.user:
+            # we don't register account if there is no user as anonymous login is then
+            # used
+            try:
+                await self.host.plugins['XEP-0077'].registerNewAccount(jid_, password)
+            except error.StanzaError as e:
+                prefix = jid_.user
+                idx = 0
+                while e.condition == 'conflict':
+                    if idx >= SUFFIX_MAX:
+                        raise exceptions.ConflictError(_("Can't create XMPP account"))
+                    jid_.user = prefix + '_' + str(idx)
+                    log.info(_("requested jid already exists, trying with {}".format(
+                        jid_.full())))
+                    try:
+                        await self.host.plugins['XEP-0077'].registerNewAccount(
+                            jid_,
+                            password
+                        )
+                    except error.StanzaError:
+                        idx += 1
+                    else:
+                        break
+                if e.condition != 'conflict':
+                    raise e
+
+            log.info(_("account {jid_} created").format(jid_=jid_.full()))
+
+        ## profile creation
+
+        extra[KEY_GUEST_PROFILE] = guest_profile = INVITEE_PROFILE_TPL.format(
+            uuid=id_
+        )
+        # profile creation should not fail as we generate unique name ourselves
+        await self.host.memory.createProfile(guest_profile, password)
+        await self.host.memory.startSession(password, guest_profile)
+        await self.host.memory.setParam("JabberID", jid_.full(), "Connection",
+                                        profile_key=guest_profile)
+        await self.host.memory.setParam("Password", password, "Connection",
+                                        profile_key=guest_profile)
+
     async def create(self, **kwargs):
         r"""Create an invitation
 
@@ -201,6 +293,13 @@ class InvitationsPlugin(object):
         self.checkExtra(extra)
 
         email = kwargs.pop('email', None)
+
+        existing = await self.getExistingInvitation(email)
+        if existing is not None:
+            log.info(f"There is already an invitation for {email!r}")
+            extra.update(existing)
+            del extra[KEY_ID]
+
         emails_extra = kwargs.pop('emails_extra', [])
         if not email and emails_extra:
             raise ValueError(
@@ -214,67 +313,18 @@ class InvitationsPlugin(object):
 
         ## uuid
         log.info(_("creating an invitation"))
-        id_ = str(shortuuid.uuid())
+        id_ = existing[KEY_ID] if existing else str(shortuuid.uuid())
 
-        ## XMPP account creation
+        if existing is None:
-        password = kwargs.pop('password', None)
+            await self._createAccountAndProfile(id_, kwargs, extra)
-        if password is None:
-           password = utils.generatePassword()
-        assert password
-        # XXX: password is here saved in clear in database
-        #      it is needed for invitation as the same password is used for profile
-        #      and SàT need to be able to automatically open the profile with the uuid
-        # FIXME: we could add an extra encryption key which would be used with the uuid
-        #        when the invitee is connecting (e.g. with URL). This key would not be
-        #        saved and could be used to encrypt profile password.
-        extra[KEY_PASSWORD] = password
 
-        jid_ = kwargs.pop('jid_', None)
+        profile = kwargs.pop('profile', None)
-        if not jid_:
+        guest_profile = extra[KEY_GUEST_PROFILE]
-            domain = self.host.memory.getConfig(None, 'xmpp_domain')
+        jid_ = jid.JID(extra[KEY_JID])
-            if not domain:
-                # TODO: fallback to profile's domain
-                raise ValueError(_("You need to specify xmpp_domain in sat.conf"))
-            jid_ = "invitation-{uuid}@{domain}".format(uuid=shortuuid.uuid(),
-                                                        domain=domain)
-        jid_ = jid.JID(jid_)
-        if jid_.user:
-            # we don't register account if there is no user as anonymous login is then
-            # used
-            try:
-                await self.host.plugins['XEP-0077'].registerNewAccount(jid_, password)
-            except error.StanzaError as e:
-                prefix = jid_.user
-                idx = 0
-                while e.condition == 'conflict':
-                    if idx >= SUFFIX_MAX:
-                        raise exceptions.ConflictError(_("Can't create XMPP account"))
-                    jid_.user = prefix + '_' + str(idx)
-                    log.info(_("requested jid already exists, trying with {}".format(
-                        jid_.full())))
-                    try:
-                        await self.host.plugins['XEP-0077'].registerNewAccount(jid_,
-                                                                               password)
-                    except error.StanzaError:
-                        idx += 1
-                    else:
-                        break
-                if e.condition != 'conflict':
-                    raise e
 
-            log.info(_("account {jid_} created").format(jid_=jid_.full()))
+        ## identity
-
-        ## profile creation
-
-        extra[KEY_GUEST_PROFILE] = guest_profile = INVITEE_PROFILE_TPL.format(uuid=id_)
-        # profile creation should not fail as we generate unique name ourselves
-        await self.host.memory.createProfile(guest_profile, password)
-        await self.host.memory.startSession(password, guest_profile)
-        await self.host.memory.setParam("JabberID", jid_.full(), "Connection",
-                                        profile_key=guest_profile)
-        await self.host.memory.setParam("Password", password, "Connection",
-                                        profile_key=guest_profile)
         name = kwargs.pop('name', None)
+        password = extra[KEY_PASSWORD]
         if name is not None:
             extra['name'] = name
             try:
@@ -306,7 +356,6 @@ class InvitationsPlugin(object):
             else:
                 format_args['name'] = name
 
-            profile = kwargs.pop('profile', None)
             if profile is None:
                 format_args['profile'] = ''
             else:
@@ -344,8 +393,6 @@ class InvitationsPlugin(object):
         if kwargs:
             log.warning(_("Not all arguments have been consumed: {}").format(kwargs))
 
-        extra[KEY_JID] = jid_.full()
-
         ## extra data saving
         self.invitations[id_] = extra
 

sat/plugins/plugin_misc_text_syntaxes.py

@@ -17,16 +17,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import re
-from html import escape
 from functools import partial
-from sat.core.i18n import _, D_
+from html import escape
-from sat.core.constants import Const as C
+import re
-from sat.core.log import getLogger
+from typing import Set
 
 from twisted.internet import defer
 from twisted.internet.threads import deferToThread
+
 from sat.core import exceptions
+from sat.core.constants import Const as C
+from sat.core.i18n import D_, _
+from sat.core.log import getLogger
 from sat.tools import xml_tools
 
 try:
@@ -115,7 +117,15 @@ VOID_ELEMENTS = (
     "track",
     "wbr")
 
-SAFE_ATTRS = html.defs.safe_attrs.union(("style", "poster", "controls"))
+SAFE_ATTRS = html.defs.safe_attrs.union({"style", "poster", "controls"}) - {"id"}
+SAFE_CLASSES = {
+    # those classes are used for code highlighting
+    "bp", "c", "ch", "cm", "cp", "cpf", "cs", "dl", "err", "fm", "gd", "ge", "get", "gh",
+    "gi", "go", "gp", "gr", "gs", "gt", "gu", "highlight", "hll", "il", "k", "kc", "kd",
+    "kn", "kp", "kr", "kt", "m", "mb", "mf", "mh", "mi", "mo", "na", "nb", "nc", "nd",
+    "ne", "nf", "ni", "nl", "nn", "no", "nt", "nv", "o", "ow", "s", "sa", "sb", "sc",
+    "sd", "se", "sh", "si", "sr", "ss", "sx", "vc", "vg", "vi", "vm", "w", "write",
+}
 STYLES_VALUES_REGEX = (
     r"^("
     + "|".join(
@@ -237,7 +247,8 @@ class TextSyntaxes(object):
         except ImportError:
             log.warning("markdown or html2text not found, can't use Markdown syntax")
             log.info(
-                "You can download/install them from https://pythonhosted.org/Markdown/ and https://github.com/Alir3z4/html2text/"
+                "You can download/install them from https://pythonhosted.org/Markdown/ "
+                "and https://github.com/Alir3z4/html2text/"
             )
         host.bridge.addMethod(
             "syntaxConvert",
@@ -288,14 +299,14 @@ class TextSyntaxes(object):
         )
         return failure
 
-    def cleanStyle(self, styles):
+    def cleanStyle(self, styles_raw: str) -> str:
         """"Clean unsafe CSS styles
 
         Remove styles not in the whitelist, or where the value doesn't match the regex
-        @param styles_raw(unicode): CSS styles
+        @param styles_raw: CSS styles
-        @return (unicode): cleaned styles
+        @return: cleaned styles
         """
-        styles = styles.split(";")
+        styles: List[str] = styles_raw.split(";")
         cleaned_styles = []
         for style in styles:
             try:
@@ -315,6 +326,14 @@ class TextSyntaxes(object):
             ["%s: %s" % (key_, value_) for key_, value_ in cleaned_styles]
         )
 
+    def cleanClasses(self, classes_raw: str) -> str:
+        """Remove any non whitelisted class
+
+        @param classes_raw: classes set on an element
+        @return: remaining classes (can be empty string)
+        """
+        return " ".join(SAFE_CLASSES.intersection(classes_raw.split()))
+
     def cleanXHTML(self, xhtml):
         """Clean XHTML text by removing potentially dangerous/malicious parts
 
@@ -341,6 +360,8 @@ class TextSyntaxes(object):
         xhtml_elt = cleaner.clean_html(xhtml_elt)
         for elt in xhtml_elt.xpath("//*[@style]"):
             elt.set("style", self.cleanStyle(elt.get("style")))
+        for elt in xhtml_elt.xpath("//*[@class]"):
+            elt.set("class", self.cleanClasses(elt.get("class")))
         # we remove self-closing elements for non-void elements
         for element in xhtml_elt.iter(tag=etree.Element):
             if not element.text: