diff --git a/doc/contribuing/index.rst b/doc/contributing/index.rst similarity index 80% rename from doc/contribuing/index.rst rename to doc/contributing/index.rst index 5a6c3a54..bcbeb3b5 100644 --- a/doc/contribuing/index.rst +++ b/doc/contributing/index.rst @@ -1,8 +1,8 @@ -.. _contribuing: +.. _contributing: -=========== -contribuing -=========== +============ +contributing +============ This part of the documentation is for people willing to contribute to the development of Libervia and its ecosystem, it is not needed to read this if you only plan to use the diff --git a/doc/contribuing/testing.rst b/doc/contributing/testing.rst similarity index 98% rename from doc/contribuing/testing.rst rename to doc/contributing/testing.rst index 080e603f..f8698aef 100644 --- a/doc/contribuing/testing.rst +++ b/doc/contributing/testing.rst @@ -28,7 +28,7 @@ end-to-end tests ================ End-to-end tests are located in ``tests/e2e``. They are launched in a well defined -environment managed through Docker. The ``docker/docker-compose_e2e.yml`` is used to +environment managed through Docker. The ``docker/docker-compose-e2e.yml`` is used to create the suitable containers. A script is available at ``tests/e2e/run_e2e.py`` to launch the tests. It will create the @@ -68,7 +68,7 @@ In addition to pytest option, some flags can be set with the following arguments To log-in into the ``backend`` container, you can use the following command, from ``/docker`` directory:: - $ docker-compose -f docker-compose_e2e.yml exec backend /bin/bash + $ docker-compose -f docker-compose-e2e.yml exec backend /bin/bash Then run a python console with given instructions @@ -224,7 +224,7 @@ test to be sure to have test profiles created and fake SMTP server run…:: …then to go into the ``backend`` container and work with the browser (to be run in ``docker`` directory)…:: - $ docker-compose -f docker-compose_e2e.yml exec backend /bin/bash + $ docker-compose -f docker-compose-e2e.yml exec backend /bin/bash …and, inside the container, you can now run ``python3`` and enter instruction prints at the end of the test session. diff --git a/doc/index.rst b/doc/index.rst index f9d18803..bbfcf530 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -29,7 +29,7 @@ You can follow this documentation to learn more on it, or join our official XMPP developer.rst /libervia-cli/index.rst /libervia-tui/index.rst - /contribuing/index.rst + /contributing/index.rst Indices and tables diff --git a/docker/backend-dev-e2e/Dockerfile b/docker/backend-dev-e2e/Dockerfile index 38325297..3866d011 100644 --- a/docker/backend-dev-e2e/Dockerfile +++ b/docker/backend-dev-e2e/Dockerfile @@ -1,4 +1,5 @@ -FROM libervia/backend:dev +ARG REVISION=dev +FROM libervia/backend:${REVISION} LABEL maintainer="Goffi " @@ -46,7 +47,7 @@ USER libervia RUN \ # pytest and its plugins pip install pytest pytest-timeout pytest-dependency \ - # needed to test jp + # needed to test libervia-cli sh \ # needed to test libervia helium \ @@ -57,7 +58,7 @@ RUN \ RUN ./entrypoint.sh \ # we create the file sharing component which will autoconnect when backend is started - jp profile create file_sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file_sharing -A && \ + li profile create file_sharing -j files.server1.test -p "" --xmpp-password test_e2e -C file_sharing -A && \ libervia-backend stop ENV LIBERVIA_TEST_REPORT_DIR=/reports diff --git a/docker/backend-dev/Dockerfile b/docker/backend-dev/Dockerfile index 2da92030..70e88c26 100644 --- a/docker/backend-dev/Dockerfile +++ b/docker/backend-dev/Dockerfile @@ -1,13 +1,15 @@ -FROM debian:buster-slim +FROM debian:bullseye-slim LABEL maintainer="Goffi " +ARG REVISION ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get upgrade -y && \ +RUN echo "Building image for Libervia ${REVISION:-dev}" && \ + apt-get update && apt-get upgrade -y && \ apt-get install -y --no-install-recommends locales python3-dev python3-venv python3-wheel mercurial libxml2-dev libxslt-dev libcairo2-dev libjpeg-dev libgirepository1.0-dev libdbus-1-dev libdbus-glib-1-dev dbus-x11 cmake && \ \ - # will be used to put many SàT specific data + # will be used to put many Libervia specific data mkdir -p /usr/share/libervia/certificates && \ addgroup tls-cert --gid 9999 && \ chown :tls-cert /usr/share/libervia/certificates && \ @@ -18,12 +20,12 @@ RUN apt-get update && apt-get upgrade -y && \ mkdir /src && chown libervia:libervia /src && \ \ # we need UTF-8 locale - sed -i "s/# en_US.UTF-8/en_US.UTF-8/" /etc/locale.gen && locale-gen && \ - \ - # we need a TCP socket for D-Bus - sed -i "s&unix:tmpdir=/tmp&\0\n tcp:host=localhost,bind=*,port=55555,family=ipv4\n ANONYMOUS\n &" /usr/share/dbus-1/session.conf + sed -i "s/# en_US.UTF-8/en_US.UTF-8/" /etc/locale.gen && locale-gen ENV LC_ALL en_US.UTF-8 +ENV DOCKER_LIBERVIA_REV=${REVISION:-dev} + +COPY --chown=root:root session.conf /usr/share/dbus-1/session.conf WORKDIR /home/libervia USER libervia @@ -34,15 +36,16 @@ RUN python3 -m venv libervia_env && libervia_env/bin/pip install -U pip wheel && # local repos without it them cause troubles hg clone https://repos.goffi.org/urwid-satext && ~/libervia_env/bin/pip install -e urwid-satext && \ mv urwid-satext/urwid_satext.egg-info ~/libervia_env/lib/python3.*/site-packages && \ - hg clone https://repos.goffi.org/sat_tmp && ~/libervia_env/bin/pip install -e sat_tmp && \ + hg clone https://repos.goffi.org/sat_tmp -u "${REVISION:-tip}" && ~/libervia_env/bin/pip install -e sat_tmp && \ mv sat_tmp/sat_tmp.egg-info ~/libervia_env/lib/python3.*/site-packages && \ - hg clone https://repos.goffi.org/sat_templates && ~/libervia_env/bin/pip install -e sat_templates && \ - mv sat_templates/libervia_templates.egg-info ~/libervia_env/lib/python3.*/site-packages && \ - hg clone https://repos.goffi.org/sat_media && \ - hg clone https://repos.goffi.org/sat && ~/libervia_env/bin/pip install -e 'sat[SVG]' && \ - mv sat/libervia_backend.egg-info ~/libervia_env/lib/python3.*/site-packages + hg clone https://repos.goffi.org/libervia-templates && ~/libervia_env/bin/pip install -e libervia-templates && \ + mv libervia-templates/libervia_templates.egg-info ~/libervia_env/lib/python3.*/site-packages && \ + hg clone https://repos.goffi.org/libervia-media && \ + hg clone https://repos.goffi.org/libervia-backend -u "${REVISION:-tip}" && ~/libervia_env/bin/pip install -e 'libervia-backend[SVG]' && \ + mv libervia-backend/libervia_backend.egg-info ~/libervia_env/lib/python3.*/site-packages && \ + mkdir -p /home/libervia/.local/share/libervia -ENV DBUS_SESSION_BUS_ADDRESS=tcp:host=backend,port=55555,family=ipv4 +ENV DBUS_SESSION_BUS_ADDRESS="tcp:host=backend,port=55555,family=ipv4;unix:path=/home/libervia/.local/share/libervia/dbus_socket" ENV PATH=/home/libervia/libervia_env/bin:$PATH COPY --chown=root:root libervia.conf /etc/_libervia.conf diff --git a/docker/backend-dev/libervia.conf b/docker/backend-dev/libervia.conf index 3e97668c..2b8a94d3 100644 --- a/docker/backend-dev/libervia.conf +++ b/docker/backend-dev/libervia.conf @@ -1,5 +1,5 @@ [DEFAULT] -media_dir = /src/sat_media +media_dir = /src/libervia-media [cli] # FIXME: default "auto" value cause trouble diff --git a/docker/backend-dev/session.conf b/docker/backend-dev/session.conf new file mode 100644 index 00000000..2fcf9b09 --- /dev/null +++ b/docker/backend-dev/session.conf @@ -0,0 +1,32 @@ + + + session + + unix:path=/home/libervia/.local/share/libervia/dbus_socket + tcp:host=localhost,bind=*,port=55555,family=ipv4 + ANONYMOUS + + + + + + + + contexts/dbus_contexts + 1000000000 + 250000000 + 1000000000 + 250000000 + 1000000000 + 120000 + 240000 + 150000 + 100000 + 10000 + 100000 + 10000 + 50000 + 50000 + 50000 + diff --git a/docker/docker-compose-e2e.yml b/docker/docker-compose-e2e.yml index b276b875..2b77fe08 100644 --- a/docker/docker-compose-e2e.yml +++ b/docker/docker-compose-e2e.yml @@ -38,8 +38,11 @@ services: LIBERVIA_PUBSUB_XMPP_PWD: test_e2e backend: - image: libervia/backend:dev-e2e - build: backend-dev-e2e + image: libervia/backend:${DOCKER_LIBERVIA_REV:-dev}-e2e + build: + context: backend-dev-e2e + args: + REVISION: ${DOCKER_LIBERVIA_REV:-} environment: LIBERVIA_TEST_ENV_E2E: "1" LIBERVIA_TEST_ENV_E2E_WEB: "1" @@ -54,8 +57,11 @@ services: - libervia-backend.test web: - image: libervia/web:dev-e2e - build: libervia-web-dev-e2e + image: libervia/web:${DOCKER_LIBERVIA_REV:-dev}-e2e + build: + context: libervia-web-dev-e2e + args: + REVISION: ${DOCKER_LIBERVIA_REV:-} depends_on: - backend environment: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a8ae3ec9..3c56d82f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,11 +2,17 @@ version: "3.6" services: backend: - image: libervia/backend:dev - build: backend-dev + image: "libervia/backend:${DOCKER_LIBERVIA_REV:-dev}" + build: + context: backend-dev + args: + REVISION: "${DOCKER_LIBERVIA_REV:-}" web: - image: libervia/web:dev - build: libervia-web-dev + image: "libervia/web:${DOCKER_LIBERVIA_REV:-dev}" + build: + context: libervia-web-dev + args: + REVISION: "${DOCKER_LIBERVIA_REV:-}" depends_on: - backend diff --git a/docker/libervia-web-dev-demo/Dockerfile b/docker/libervia-web-dev-demo/Dockerfile index 8f9b97c8..6b00e7a9 100644 --- a/docker/libervia-web-dev-demo/Dockerfile +++ b/docker/libervia-web-dev-demo/Dockerfile @@ -1,4 +1,5 @@ -FROM libervia/web:dev +ARG REVISION=dev +FROM libervia/web:${REVISION} LABEL maintainer="Goffi " diff --git a/docker/libervia-web-dev-e2e/Dockerfile b/docker/libervia-web-dev-e2e/Dockerfile index 15f14757..20647eb3 100644 --- a/docker/libervia-web-dev-e2e/Dockerfile +++ b/docker/libervia-web-dev-e2e/Dockerfile @@ -1,4 +1,5 @@ -FROM libervia/web:dev +ARG REVISION=dev +FROM libervia/web:${REVISION} LABEL maintainer="Goffi " diff --git a/docker/libervia-web-dev/Dockerfile b/docker/libervia-web-dev/Dockerfile index d3f83d6f..390f9479 100644 --- a/docker/libervia-web-dev/Dockerfile +++ b/docker/libervia-web-dev/Dockerfile @@ -1,7 +1,9 @@ -FROM libervia/backend:dev +ARG REVISION +FROM libervia/backend:${REVISION:-dev} LABEL maintainer="Goffi " +ARG REVISION ARG DEBIAN_FRONTEND=noninteractive USER root @@ -9,9 +11,9 @@ USER root RUN apt-get install -y --no-install-recommends yarnpkg WORKDIR /home/libervia USER libervia -RUN cd /src && hg clone https://repos.goffi.org/libervia && \ - ~/libervia_env/bin/pip install -e libervia && \ - mv libervia/libervia_web.egg-info ~/libervia_env/lib/python3.*/site-packages +RUN cd /src && hg clone https://repos.goffi.org/libervia-web -u "${REVISION:-tip}" && \ + ~/libervia_env/bin/pip install -e libervia-web && \ + mv libervia-web/libervia_web.egg-info ~/libervia_env/lib/python3.*/site-packages RUN ./entrypoint.sh \ # we build here to avoid re-downloading node modules or other browser diff --git a/docker/pubsub/Dockerfile b/docker/pubsub/Dockerfile index 8a307890..637c9749 100644 --- a/docker/pubsub/Dockerfile +++ b/docker/pubsub/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster-slim +FROM debian:bullseye-slim LABEL maintainer="Goffi " @@ -29,6 +29,6 @@ RUN python3 -m venv libervia_env && libervia_env/bin/pip install -U pip wheel && hg clone https://repos.goffi.org/sat_tmp && ~/libervia_env/bin/pip install -e sat_tmp && \ mv sat_tmp/sat_tmp.egg-info ~/libervia_env/lib/python3.*/site-packages && \ hg clone https://repos.goffi.org/sat_pubsub && ~/libervia_env/bin/pip install -e sat_pubsub && \ - mv sat_pubsub/libervia_pubsub.egg-info ~/libervia_env/lib/python3.*/site-packages + mv sat_pubsub/sat_pubsub.egg-info ~/libervia_env/lib/python3.*/site-packages ENTRYPOINT ["/home/libervia/entrypoint.sh"] diff --git a/sat/bridge/bridge_constructor/constructors/dbus/dbus_frontend_template.py b/sat/bridge/bridge_constructor/constructors/dbus/dbus_frontend_template.py index a8addfd1..877f00bc 100644 --- a/sat/bridge/bridge_constructor/constructors/dbus/dbus_frontend_template.py +++ b/sat/bridge/bridge_constructor/constructors/dbus/dbus_frontend_template.py @@ -140,7 +140,21 @@ class Bridge: kwargs['reply_handler'] = _callback kwargs['error_handler'] = lambda err: _errback(dbus_to_bridge_exception(err)) - return method(*args, **kwargs) + try: + return method(*args, **kwargs) + except ValueError as e: + if e.args[0].startswith("Unable to guess signature"): + # XXX: if frontend is started too soon after backend, the + # inspection misses methods (notably plugin dynamically added + # methods). The following hack works around that by redoing the + # cache of introspected methods signatures. + log.debug("using hack to work around inspection issue") + proxy = self.db_plugin_iface.proxy_object + IN_PROGRESS = proxy.INTROSPECT_STATE_INTROSPECT_IN_PROGRESS + proxy._introspect_state = IN_PROGRESS + proxy._Introspect() + return self.db_plugin_iface.get_dbus_method(name)(*args, **kwargs) + raise e return getPluginMethod @@ -167,13 +181,32 @@ class AIOBridge(Bridge): fut.set_result, ret) error_handler = lambda err: loop.call_soon_threadsafe( fut.set_exception, dbus_to_bridge_exception(err)) - method( - *args, - **kwargs, - timeout=const_TIMEOUT, - reply_handler=reply_handler, - error_handler=error_handler - ) + try: + method( + *args, + **kwargs, + timeout=const_TIMEOUT, + reply_handler=reply_handler, + error_handler=error_handler + ) + except ValueError as e: + if e.args[0].startswith("Unable to guess signature"): + # same hack as for Bridge.__getattribute__ + log.warning("using hack to work around inspection issue") + proxy = self.db_plugin_iface.proxy_object + IN_PROGRESS = proxy.INTROSPECT_STATE_INTROSPECT_IN_PROGRESS + proxy._introspect_state = IN_PROGRESS + proxy._Introspect() + self.db_plugin_iface.get_dbus_method(name)( + *args, + **kwargs, + timeout=const_TIMEOUT, + reply_handler=reply_handler, + error_handler=error_handler + ) + + else: + raise e return fut return getPluginMethod diff --git a/sat/core/sat_main.py b/sat/core/sat_main.py index c35e0cb1..56731f8e 100644 --- a/sat/core/sat_main.py +++ b/sat/core/sat_main.py @@ -228,11 +228,7 @@ class SAT(service.Service): ui_contact_list.ContactList(self) ui_profile_manager.ProfileManager(self) except Exception as e: - log.error( - _("Could not initialize backend: {reason}").format( - reason=str(e).decode("utf-8", "ignore") - ) - ) + log.error(f"Could not initialize backend: {e}") sys.exit(1) self._addBaseMenus() @@ -344,6 +340,12 @@ class SAT(service.Service): plugin_modes = plugin_info["modes"] = set( plugin_info.setdefault("modes", C.PLUG_MODE_DEFAULT) ) + if not plugin_modes.intersection(C.PLUG_MODE_BOTH): + log.error( + f"Can't import plugin at {plugin_path}, invalid {C.PI_MODES!r} " + f"value: {plugin_modes!r}" + ) + continue # if the plugin is an entry point, it must work in component mode if plugin_info["type"] == C.PLUG_TYPE_ENTRY_POINT: diff --git a/sat/memory/persistent.py b/sat/memory/persistent.py index a4d02e32..0bad2089 100644 --- a/sat/memory/persistent.py +++ b/sat/memory/persistent.py @@ -204,7 +204,7 @@ class LazyPersistentBinaryDict(PersistentBinaryDict): )) def __repr__(self): - raise NotImplementedError + return self.__str__() def __str__(self): return "LazyPersistentBinaryDict (namespace: {})".format(self.namespace) diff --git a/sat/plugins/plugin_misc_android.py b/sat/plugins/plugin_misc_android.py index 085c5c42..d48ff3a8 100644 --- a/sat/plugins/plugin_misc_android.py +++ b/sat/plugins/plugin_misc_android.py @@ -309,9 +309,11 @@ class AndroidPlugin(object): # "XXX has no attribute 'invoke'" error (looks like the same issue as # https://github.com/kivy/pyjnius/issues/59) self.br = BroadcastReceiver( - callback=lambda *args, **kwargs: reactor.callLater(0, - self.onConnectivityChange), - actions=["android.net.conn.CONNECTIVITY_CHANGE"]) + callback=lambda *args, **kwargs: reactor.callFromThread( + self.onConnectivityChange + ), + actions=["android.net.conn.CONNECTIVITY_CHANGE"] + ) self.br.start() @property diff --git a/sat/plugins/plugin_misc_file.py b/sat/plugins/plugin_misc_file.py index 4f6c2457..54d1b69b 100644 --- a/sat/plugins/plugin_misc_file.py +++ b/sat/plugins/plugin_misc_file.py @@ -237,8 +237,8 @@ class FilePlugin: data_cb=file_data.get("data_cb"), ) - def _gotConfirmation( - self, data, client, peer_jid, transfer_data, file_data, stream_object + async def _gotConfirmation( + self, client, data, peer_jid, transfer_data, file_data, stream_object ): """Called when the permission and dest path have been received @@ -258,19 +258,7 @@ class FilePlugin: # we manage case where file already exists if os.path.exists(file_path): - - def check_overwrite(overwrite): - if overwrite: - self.openFileWrite( - client, file_path, transfer_data, file_data, stream_object - ) - return True - else: - return defer.ensureDeferred( - self.getDestDir(client, peer_jid, transfer_data, file_data) - ) - - exists_d = xml_tools.deferConfirm( + overwrite = await xml_tools.deferConfirm( self.host, _(CONFIRM_OVERWRITE).format(file_path), _(CONFIRM_OVERWRITE_TITLE), @@ -282,8 +270,9 @@ class FilePlugin: security_limit=SECURITY_LIMIT, profile=client.profile, ) - exists_d.addCallback(check_overwrite) - return exists_d + + if not overwrite: + return await self.getDestDir(client, peer_jid, transfer_data, file_data) self.openFileWrite(client, file_path, transfer_data, file_data, stream_object) return True @@ -315,7 +304,7 @@ class FilePlugin: "size_human" will also be added with human readable file size @param stream_object(bool): if True, a stream_object will be used instead of file_obj a stream.FileStreamObject will be used - return (defer.Deferred): True if transfer is accepted + return: True if transfer is accepted """ cont, ret_value = await self.host.trigger.asyncReturnPoint( "FILE_getDestDir", client, peer_jid, transfer_data, file_data, stream_object @@ -327,7 +316,7 @@ class FilePlugin: assert PROGRESS_ID_KEY in file_data # human readable size file_data["size_human"] = common_utils.getHumanSize(file_data["size"]) - d = xml_tools.deferDialog( + resp_data = await xml_tools.deferDialog( self.host, _(CONFIRM).format(peer=peer_jid.full(), **file_data), _(CONFIRM_TITLE), @@ -341,12 +330,13 @@ class FilePlugin: security_limit=SECURITY_LIMIT, profile=client.profile, ) - d.addCallback( - self._gotConfirmation, + + accepted = await self._gotConfirmation( client, + resp_data, peer_jid, transfer_data, file_data, stream_object, ) - return d + return accepted diff --git a/sat/plugins/plugin_xep_0277.py b/sat/plugins/plugin_xep_0277.py index 5fa90a03..e1022c0d 100644 --- a/sat/plugins/plugin_xep_0277.py +++ b/sat/plugins/plugin_xep_0277.py @@ -19,7 +19,6 @@ import time import dateutil import calendar -import urllib.parse from secrets import token_urlsafe from typing import Optional from functools import partial @@ -1032,13 +1031,12 @@ class XEP_0277(object): will return(JID(u'sat-pubsub.example.net'), 'urn:xmpp:comments:_af43b363-3259-4b2a-ba4c-1bc33aa87634__urn:xmpp:groupblog:somebody@example.net') @return (tuple[jid.JID, unicode]): service and node """ - parsed_url = urllib.parse.urlparse(node_url, "xmpp") - service = jid.JID(parsed_url.path) - parsed_queries = urllib.parse.parse_qs(parsed_url.query) - node = parsed_queries.get("node", [""])[0] - - if not node: - raise exceptions.DataError("Invalid comments link") + try: + parsed_url = xmpp_uri.parseXMPPUri(node_url) + service = jid.JID(parsed_url["path"]) + node = parsed_url["node"] + except Exception as e: + raise exceptions.DataError(f"Invalid comments link: {e}") return (service, node) diff --git a/sat/plugins/plugin_xep_0353.py b/sat/plugins/plugin_xep_0353.py index 6dea3783..fd24856f 100644 --- a/sat/plugins/plugin_xep_0353.py +++ b/sat/plugins/plugin_xep_0353.py @@ -19,7 +19,7 @@ from zope.interface import implementer from twisted.internet import defer from twisted.internet import reactor -from twisted.words.protocols.jabber import xmlstream, jid +from twisted.words.protocols.jabber import xmlstream, jid, error from twisted.words.xish import domish from wokkel import disco, iwokkel from sat.core.i18n import _, D_ @@ -38,7 +38,7 @@ PLUGIN_INFO = { C.PI_NAME: "Jingle Message Initiation", C.PI_IMPORT_NAME: "XEP-0353", C.PI_TYPE: "XEP", - C.PI_MODES: C.PLUG_MODE_CLIENT, + C.PI_MODES: [C.PLUG_MODE_CLIENT], C.PI_PROTOCOLS: ["XEP-0353"], C.PI_DEPENDENCIES: ["XEP-0166"], C.PI_MAIN: "XEP_0353", @@ -85,8 +85,15 @@ class XEP_0353: if peer_jid.resource: return True - infos = await self.host.memory.disco.getInfos(client, peer_jid) - categories = {c for c, __ in infos.identities} + try: + infos = await self.host.memory.disco.getInfos(client, peer_jid) + except error.StanzaError as e: + if e.condition == "service-unavailable": + categories = {} + else: + raise e + else: + categories = {c for c, __ in infos.identities} if "component" in categories: # we don't use message initiation with components return True diff --git a/sat/tools/common/uri.py b/sat/tools/common/uri.py index ffbdcbcc..5ff54c1e 100644 --- a/sat/tools/common/uri.py +++ b/sat/tools/common/uri.py @@ -19,6 +19,7 @@ """ XMPP uri parsing tools """ +import sys import urllib.parse import urllib.request, urllib.parse, urllib.error @@ -53,7 +54,13 @@ def parseXMPPUri(uri): if query_end == -1 or "=" in query_type: raise ValueError("no query type, invalid XMPP URI") - pairs = urllib.parse.parse_qs(uri_split.geturl()) + if sys.version_info >= (3, 9): + # parse_qs behaviour has been modified in Python 3.9, ";" is not understood as a + # parameter separator anymore but the "separator" argument has been added to + # change it. + pairs = urllib.parse.parse_qs(uri_split.geturl(), separator=";") + else: + pairs = urllib.parse.parse_qs(uri_split.geturl()) for k, v in list(pairs.items()): if len(v) != 1: raise NotImplementedError("multiple values not managed") diff --git a/sat_frontends/bridge/dbus_bridge.py b/sat_frontends/bridge/dbus_bridge.py index b250607d..14926b1f 100644 --- a/sat_frontends/bridge/dbus_bridge.py +++ b/sat_frontends/bridge/dbus_bridge.py @@ -140,7 +140,21 @@ class Bridge: kwargs['reply_handler'] = _callback kwargs['error_handler'] = lambda err: _errback(dbus_to_bridge_exception(err)) - return method(*args, **kwargs) + try: + return method(*args, **kwargs) + except ValueError as e: + if e.args[0].startswith("Unable to guess signature"): + # XXX: if frontend is started too soon after backend, the + # inspection misses methods (notably plugin dynamically added + # methods). The following hack works around that by redoing the + # cache of introspected methods signatures. + log.debug("using hack to work around inspection issue") + proxy = self.db_plugin_iface.proxy_object + IN_PROGRESS = proxy.INTROSPECT_STATE_INTROSPECT_IN_PROGRESS + proxy._introspect_state = IN_PROGRESS + proxy._Introspect() + return self.db_plugin_iface.get_dbus_method(name)(*args, **kwargs) + raise e return getPluginMethod @@ -930,13 +944,32 @@ class AIOBridge(Bridge): fut.set_result, ret) error_handler = lambda err: loop.call_soon_threadsafe( fut.set_exception, dbus_to_bridge_exception(err)) - method( - *args, - **kwargs, - timeout=const_TIMEOUT, - reply_handler=reply_handler, - error_handler=error_handler - ) + try: + method( + *args, + **kwargs, + timeout=const_TIMEOUT, + reply_handler=reply_handler, + error_handler=error_handler + ) + except ValueError as e: + if e.args[0].startswith("Unable to guess signature"): + # same hack as for Bridge.__getattribute__ + log.warning("using hack to work around inspection issue") + proxy = self.db_plugin_iface.proxy_object + IN_PROGRESS = proxy.INTROSPECT_STATE_INTROSPECT_IN_PROGRESS + proxy._introspect_state = IN_PROGRESS + proxy._Introspect() + self.db_plugin_iface.get_dbus_method(name)( + *args, + **kwargs, + timeout=const_TIMEOUT, + reply_handler=reply_handler, + error_handler=error_handler + ) + + else: + raise e return fut return getPluginMethod diff --git a/sat_frontends/jp/constants.py b/sat_frontends/jp/constants.py index 1a2e8927..223d5706 100644 --- a/sat_frontends/jp/constants.py +++ b/sat_frontends/jp/constants.py @@ -54,6 +54,11 @@ class Const(constants.Const): OUTPUT_XMLUI, OUTPUT_LIST_XMLUI, ) + OUTPUT_NAME_SIMPLE = "simple" + OUTPUT_NAME_XML = "xml" + OUTPUT_NAME_XML_RAW = "xml-raw" + OUTPUT_NAME_JSON = "json" + OUTPUT_NAME_JSON_RAW = "json-raw" # Pubsub options flags SERVICE = "service" # service required diff --git a/sat_frontends/jp/output_std.py b/sat_frontends/jp/output_std.py index baa60d2d..24ab8fdf 100644 --- a/sat_frontends/jp/output_std.py +++ b/sat_frontends/jp/output_std.py @@ -26,9 +26,6 @@ from sat.tools.common import date_utils import json __outputs__ = ["Simple", "Json"] -SIMPLE = "simple" -JSON = "json" -JSON_RAW = "json_raw" class Simple(object): @@ -36,13 +33,13 @@ class Simple(object): def __init__(self, host): self.host = host - host.register_output(C.OUTPUT_TEXT, SIMPLE, self.simple_print) - host.register_output(C.OUTPUT_LIST, SIMPLE, self.list) - host.register_output(C.OUTPUT_DICT, SIMPLE, self.dict) - host.register_output(C.OUTPUT_LIST_DICT, SIMPLE, self.list_dict) - host.register_output(C.OUTPUT_DICT_DICT, SIMPLE, self.dict_dict) - host.register_output(C.OUTPUT_MESS, SIMPLE, self.messages) - host.register_output(C.OUTPUT_COMPLEX, SIMPLE, self.simple_print) + host.register_output(C.OUTPUT_TEXT, C.OUTPUT_NAME_SIMPLE, self.simple_print) + host.register_output(C.OUTPUT_LIST, C.OUTPUT_NAME_SIMPLE, self.list) + host.register_output(C.OUTPUT_DICT, C.OUTPUT_NAME_SIMPLE, self.dict) + host.register_output(C.OUTPUT_LIST_DICT, C.OUTPUT_NAME_SIMPLE, self.list_dict) + host.register_output(C.OUTPUT_DICT_DICT, C.OUTPUT_NAME_SIMPLE, self.dict_dict) + host.register_output(C.OUTPUT_MESS, C.OUTPUT_NAME_SIMPLE, self.messages) + host.register_output(C.OUTPUT_COMPLEX, C.OUTPUT_NAME_SIMPLE, self.simple_print) def simple_print(self, data): self.host.disp(str(data)) @@ -109,19 +106,19 @@ class Json(object): def __init__(self, host): self.host = host - host.register_output(C.OUTPUT_TEXT, JSON, self.dump) - host.register_output(C.OUTPUT_LIST, JSON, self.dump_pretty) - host.register_output(C.OUTPUT_LIST, JSON_RAW, self.dump) - host.register_output(C.OUTPUT_DICT, JSON, self.dump_pretty) - host.register_output(C.OUTPUT_DICT, JSON_RAW, self.dump) - host.register_output(C.OUTPUT_LIST_DICT, JSON, self.dump_pretty) - host.register_output(C.OUTPUT_LIST_DICT, JSON_RAW, self.dump) - host.register_output(C.OUTPUT_DICT_DICT, JSON, self.dump_pretty) - host.register_output(C.OUTPUT_DICT_DICT, JSON_RAW, self.dump) - host.register_output(C.OUTPUT_MESS, JSON, self.dump_pretty) - host.register_output(C.OUTPUT_MESS, JSON_RAW, self.dump) - host.register_output(C.OUTPUT_COMPLEX, JSON, self.dump_pretty) - host.register_output(C.OUTPUT_COMPLEX, JSON_RAW, self.dump) + host.register_output(C.OUTPUT_TEXT, C.OUTPUT_NAME_JSON, self.dump) + host.register_output(C.OUTPUT_LIST, C.OUTPUT_NAME_JSON, self.dump_pretty) + host.register_output(C.OUTPUT_LIST, C.OUTPUT_NAME_JSON_RAW, self.dump) + host.register_output(C.OUTPUT_DICT, C.OUTPUT_NAME_JSON, self.dump_pretty) + host.register_output(C.OUTPUT_DICT, C.OUTPUT_NAME_JSON_RAW, self.dump) + host.register_output(C.OUTPUT_LIST_DICT, C.OUTPUT_NAME_JSON, self.dump_pretty) + host.register_output(C.OUTPUT_LIST_DICT, C.OUTPUT_NAME_JSON_RAW, self.dump) + host.register_output(C.OUTPUT_DICT_DICT, C.OUTPUT_NAME_JSON, self.dump_pretty) + host.register_output(C.OUTPUT_DICT_DICT, C.OUTPUT_NAME_JSON_RAW, self.dump) + host.register_output(C.OUTPUT_MESS, C.OUTPUT_NAME_JSON, self.dump_pretty) + host.register_output(C.OUTPUT_MESS, C.OUTPUT_NAME_JSON_RAW, self.dump) + host.register_output(C.OUTPUT_COMPLEX, C.OUTPUT_NAME_JSON, self.dump_pretty) + host.register_output(C.OUTPUT_COMPLEX, C.OUTPUT_NAME_JSON_RAW, self.dump) def dump(self, data): self.host.disp(json.dumps(data, default=str)) diff --git a/sat_frontends/jp/output_xml.py b/sat_frontends/jp/output_xml.py index a67741de..29131f94 100644 --- a/sat_frontends/jp/output_xml.py +++ b/sat_frontends/jp/output_xml.py @@ -1,7 +1,6 @@ #! /usr/bin/env python3 - -# jp: a SàT command line tool +# Libervia CLI frontend # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) # This program is free software: you can redistribute it and/or modify @@ -36,8 +35,6 @@ except ImportError: __outputs__ = ["XML"] -RAW = "xml_raw" -PRETTY = "xml_pretty" class XML(object): @@ -45,16 +42,20 @@ class XML(object): def __init__(self, host): self.host = host - host.register_output(C.OUTPUT_XML, PRETTY, self.pretty, default=True) - host.register_output(C.OUTPUT_LIST_XML, PRETTY, self.pretty_list, default=True) - host.register_output(C.OUTPUT_XML, RAW, self.raw) - host.register_output(C.OUTPUT_LIST_XML, RAW, self.list_raw) + host.register_output(C.OUTPUT_XML, C.OUTPUT_NAME_XML, self.pretty, default=True) + host.register_output( + C.OUTPUT_LIST_XML, C.OUTPUT_NAME_XML, self.pretty_list, default=True + ) + host.register_output(C.OUTPUT_XML, C.OUTPUT_NAME_XML_RAW, self.raw) + host.register_output(C.OUTPUT_LIST_XML, C.OUTPUT_NAME_XML_RAW, self.list_raw) def colorize(self, xml): if pygments is None: self.host.disp( _( - "Pygments is not available, syntax highlighting is not possible. Please install if from http://pygments.org or with pip install pygments" + "Pygments is not available, syntax highlighting is not possible. " + "Please install if from http://pygments.org or with pip install " + "pygments" ), error=True, ) diff --git a/tests/e2e/libervia-cli/conftest.py b/tests/e2e/libervia-cli/conftest.py index 9f6f56a8..362559fd 100644 --- a/tests/e2e/libervia-cli/conftest.py +++ b/tests/e2e/libervia-cli/conftest.py @@ -35,7 +35,7 @@ class LiberviaCliJson: def __call__(self, *args, **kwargs): args = self.subcommands + list(args) self.subcommands.clear() - kwargs['output'] = 'json_raw' + kwargs['output'] = 'json-raw' kwargs['_tty_out'] = False cmd = li(*args, **kwargs) return json.loads(cmd.stdout) @@ -60,7 +60,7 @@ class LiberviaCliElt(LiberviaCliJson): def __call__(self, *args, **kwargs): args = self.subcommands + list(args) self.subcommands.clear() - kwargs['output'] = 'xml_raw' + kwargs['output'] = 'xml-raw' kwargs['_tty_out'] = False cmd = li(*args, **kwargs) return self.parser(cmd.stdout.decode().strip()) @@ -153,13 +153,13 @@ class Editor: @pytest.fixture(scope="session") def li_json(): - """Run li with "json_raw" output, and returns the parsed value""" + """Run li with "json-raw" output, and returns the parsed value""" return LiberviaCliJson() @pytest.fixture(scope="session") def li_elt(): - """Run li with "xml_raw" output, and returns the parsed value""" + """Run li with "xml-raw" output, and returns the parsed value""" return LiberviaCliElt() diff --git a/tests/e2e/libervia-cli/test_libervia-cli.py b/tests/e2e/libervia-cli/test_libervia-cli.py index 90fd756b..93235ddc 100644 --- a/tests/e2e/libervia-cli/test_libervia-cli.py +++ b/tests/e2e/libervia-cli/test_libervia-cli.py @@ -176,7 +176,7 @@ class TestLiberviaCliBlog: li.blog.edit(_env=editor.env) assert len(editor.original_content) == 0 assert editor.new_content == payload_md - items_data = li_json.blog.get(max=1) + items_data = li_json.blog.get(max_items=1) last_item = items_data[0][0] last_item_id = last_item['id'] assert last_item['content'] == "content in markdown" @@ -187,7 +187,7 @@ class TestLiberviaCliBlog: li.blog.edit("--last-item", _env=editor.env) assert editor.original_content == payload_md assert editor.new_content == f"{payload_md} extended" - items_data = li_json.blog.get(max=1) + items_data = li_json.blog.get(max_items=1) last_item = items_data[0][0] # we check that the id hasn't been modified assert last_item['id'] == last_item_id diff --git a/tests/e2e/libervia-web/test_libervia-web.py b/tests/e2e/libervia-web/test_libervia-web.py index 3fae8a85..205dbffe 100644 --- a/tests/e2e/libervia-web/test_libervia-web.py +++ b/tests/e2e/libervia-web/test_libervia-web.py @@ -94,9 +94,9 @@ class TestPhotos: click(album_link) wait_until(lambda: S("#file_drop").exists()) wait_until(lambda: not S("#loading_screen").exists()) - drag_file("/src/sat/tests/_files/test_1.jpg", "drop photos here") + drag_file("/src/libervia-backend/tests/_files/test_1.jpg", "drop photos here") wait_until(lambda: len(find_all(S("div.progress_finished")))==1) - drag_file("/src/sat/tests/_files/test_2.jpg", "drop photos here") + drag_file("/src/libervia-backend/tests/_files/test_2.jpg", "drop photos here") wait_until(lambda: len(find_all(S("div.progress_finished")))==2) assert S('img[alt="test_1.jpg"]').exists() assert S('img[alt="test_2.jpg"]').exists() diff --git a/tests/e2e/run_e2e.py b/tests/e2e/run_e2e.py index d47682c5..db9b3684 100755 --- a/tests/e2e/run_e2e.py +++ b/tests/e2e/run_e2e.py @@ -158,19 +158,19 @@ def use_e2e_env(): libervia_web_root_path = libervia_web_path.parent if (libervia_web_root_path / ".hg").is_dir(): libervia_web_source = libervia_web_root_path - libervia_web_target = "/src/libervia" + libervia_web_target = "/src/libervia-web" else: libervia_web_source = libervia_web_path - libervia_web_target = "/src/libervia/libervia" + libervia_web_target = "/src/libervia-web/libervia" sat_templates_path = Path(sat_templates.__file__).parent.resolve() sat_templates_root_path = sat_templates_path.parent if (sat_templates_root_path / ".hg").is_dir(): sat_templates_source = sat_templates_root_path - sat_templates_target = "/src/sat_templates" + sat_templates_target = "/src/libervia-templates" else: sat_templates_source = sat_templates_path - sat_templates_target = "/src/sat_templates/sat_templates" + sat_templates_target = "/src/libervia-templates/sat_templates" compose_e2e_path = docker_path / "docker-compose-e2e.yml" if not compose_e2e_path.is_file(): @@ -186,13 +186,13 @@ def use_e2e_env(): volumes: - type: bind source: {backend_root_path} - target: /src/sat + target: /src/libervia-backend read_only: true web: volumes: - type: bind source: {backend_root_path} - target: /src/sat + target: /src/libervia-backend read_only: true - type: bind source: {libervia_web_source} @@ -224,7 +224,7 @@ def use_e2e_env(): docker_compose.up("-d") p = docker_compose.exec( - "-T", "--workdir", "/src/sat/tests", "backend", + "-T", "--workdir", "/src/libervia-backend/tests", "backend", "pytest", "-o", "cache_dir=/tmp", *sys.argv[1:], color="yes", _in=sys.stdin, _out=live_out, _out_bufsize=0, _err=live_err, _err_bufsize=0, _bg=True