merge bookmark 0.9

dev-requirements.txt

@@ -0,0 +1,4 @@
+-r requirements.txt
+
+pytest
+pytest_twisted

doc/_ext/docstring.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+"""Adapt Libervia docstring style to autodoc"""
+
+
+def process_docstring(app, what, name, obj, options, lines):
+    lines[:] = [
+        l.replace("@param", ":param").replace("@raise", ":raises")
+        for l in lines
+    ]
+
+
+def setup(app):
+    app.connect("autodoc-process-docstring", process_docstring)
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

doc/components.rst

@@ -151,3 +151,107 @@ All options are to be set in ``[component file-sharing]`` section.
     }
 
   .. _SI prefixes and binary prefixes: https://en.wikipedia.org/wiki/Octet_(computing)#Unit_multiples
+
+
+ActivityPub Gateway
+-------------------
+
+**entry_point:** ``ap-gateway``
+
+.. note::
+
+  this component is currently in active development, and not yet usable for end-user. This
+  documentation will be updated during evolution of component.
+
+  You can follow the development by reading `Libervia Progress Notes`_.
+
+  .. _Libervia Progress Notes: https://www.goffi.org/tag/Libervia%20progress
+
+This gateway will provide a bidirectional gateway between XMPP and `ActivityPub`_ (or AP
+below). That means that user from XMPP will be able to follow actors or comments messages
+from any software compatible with ActivityPub protocol, and vice versa.
+
+.. _ActivityPub: https://activitypub.rocks/
+
+.. note::
+
+  this component is mostly tested with Prosody as XMPP server reference, and Mastodon as
+  AP server reference, but it should work with any XMPP or AP server.
+
+The component launches a HTTP server (necessary to communicate with AP software). This
+server needs to handle HTTP requests made at paths ``/.well-known/webfinger`` and ``/_ap``
+(or the ``ap_path`` set in configuration, see below). If the component is not directly
+facing internet (e.g. integrated in an existing website though a proxy), you'll have to
+redirect the requests made to those path to the HTTP server (i.e. to component host at the
+port set at ``http_port``, see configuration below). Please check your HTTP server
+documentation to find how this must be done.
+
+how to address an AP actor from XMPP
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When addressing an ActivityPub actor from XMPP, you must use a JID corresponding to the
+actor. The domain part of the JID correspond to the gateway JID (the one set in gateway
+profile), while the local part (before the ``@``) is used to specify the AP actor.
+
+`XEP-0106`_ (JID Escaping) is used to indicate the AP actor identifier, thus the ``@``
+must be escaped with ``\40``.
+
+.. _XEP-0106: JID Escaping
+
+**example**
+
+If Louise wants to talk to Pierre which is on the ``example.net`` AP server, she can use
+her XMPP AP gateway which is at ``ap.example.org``. Pierre AP's actor identifier is
+``pierre@example.net``, Louise can access it via the JID
+``pierre\40example.net@ap.example.org``.
+
+Of course, this is a bit cumbersome to do by hand, it is expected that XMPP clients will
+do the (un)escaping automatically for end-user, in a way that Louise could enter
+``pierre@example.net`` directly, with an indicator to show that this is an ActivityPub
+actor identifier rather than an XMPP JID.
+
+how to address an XMPP entity from AP
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To access an XMPP entity, just use its bare JID as AP actor.
+
+**example**
+
+If Pierre wants to talk Louise, he can directly use the JID which is the same as the AP
+actor identifier, i.e. ``louise@example.org`` (note that on AP software, a ``@`` prefix is
+often required, thus Pierre will look for ``@louise@example.org``).
+
+.. note::
+
+   Currently the gateway only uses bare JID, and character set must be recognised by the
+   AP software to be used. This will evolve in the close future to have a way to access
+   various XMPP Pubsub nodes.
+
+   The actor endpoint can also be used directly in AP software (in the example above, it
+   would be by default ``https://example.org/_ap/actor/louise%40example.org``).
+
+
+using the component
+~~~~~~~~~~~~~~~~~~~
+
+The component is currently only usable for development purpose, and it can be used with
+the following method (with can be accessed through the ``APSend`` bridge method, client is
+then replaced by the ``profile`` name, as last argument):
+
+.. automethod:: sat.plugins.plugin_comp_ap_gateway.APGateway.publishMessage
+
+The method can be used either with CLI's :ref:`debug bridge method
+<li_debug_bridge_method>` or with any D-Bus tool like ``qdbus`` or ``d-feet`` (only if you
+use the D-Bus bridge).
+
+example
+~~~~~~~
+
+On its ``example.net`` Mastodon instance, Pierre has published a message with the id
+``https://example.net/@pierre/106986412193109832``. To send a reply to this message,
+Louise can use the following command::
+
+  $ li debug bridge method -c APSend '"{\"node\": \"https://example.net/@pierre/106986412193109832\", \"content\": \"A lille hello from XMPP\"}","pierre\\40example.net@ap.example.org", "louise"'
+
+Note the double escaping, one for the shell argument, and the other to specify JSON
+object.

doc/conf.py

@@ -12,13 +12,15 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+import os
+import sys
 import os.path
 import re
 
 
+sys.path.insert(0, os.path.abspath("./_ext"))
+
+
 # -- Project information -----------------------------------------------------
 
 project = 'Libervia'
@@ -47,6 +49,8 @@ release = version_full
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    "sphinx.ext.autodoc",
+    "docstring"
 ]
 
 # Add any paths that contain templates here, relative to this directory.

doc/configuration.rst

@@ -108,6 +108,8 @@ the comments to get the meaning of each option.
     background = auto
     # end-user facing URL, used by Web frontend
     public_url = example.com
+    # uncomment next line if you don't want to use local cache for pubsub items
+    ; pubsub_cache_strategy = "no_cache"
 
     [plugin account]
     ; where a new account must be created

doc/developer.rst

@@ -0,0 +1,82 @@
+.. _developer:
+
+=======================
+Developer Documentation
+=======================
+
+This documentation is intended for people who wants to contribute or work with the
+internals of the project, it is not for end-users.
+
+Storage
+=======
+
+Since version 0.9, Libervia uses SQLAlchemy_ with its Object–Relational Mapping as a
+backend to store persistent data, and Alembic_ is used to handle schema and data
+migrations.
+
+SQLite_ is currently the only supported database, but it is planned to add support for
+other ones (notably PostgreSQL), probably during the development of 0.9 version.
+
+The mapping is done in ``sat.memory.sqla_mapping`` and working with database is done
+through high level methods found in ``sat.memory.sqla``.
+
+Before the move to SQLAlchemy, there was a strict separation between database
+implementation and the rest of the code. With 0.9, objects mapping to database can be used
+and manipulated directly outside of ``sat.memory.sqla`` to take profit of SQLAlchemy
+possibilities.
+
+Database state is detected when the backend starts, and the database will be created or
+migrated automatically if necessary.
+
+To create a new migration script, ``Alembic`` may be used directly. To do so, be sure to
+have an up-to-date database (and a backup in case of troubles), then activate the virtual
+environment where Libervia is installed (Alembic needs to access ORM mapping), go to
+``sat/memory/migration`` directory, and enter the following command::
+
+  alembic revision --autogenerate -m "some revision message"
+
+This will create a base migration file in ``versions`` directory. Adapt it to your needs,
+try to create both ``upgrade`` and ``downgrade`` method whenever possible, and be sure to
+test it in both directions (``alembic upgrade head`` and ``alembic downgrade
+<previous_revision>``). Please check Alembic documentation for more details.
+
+.. _SQLALchemy: https://www.sqlalchemy.org/
+.. _Alembic: https://alembic.sqlalchemy.org/
+.. _SQLite: https://sqlite.org
+
+Pubsub Cache
+============
+
+There is an internal cache for pubsub nodes and items, which is done in
+``plugin_pubsub_cache``. The ``PubsubNode`` and ``PubsubItem`` class are the one mapping
+the database.
+
+The cache is operated transparently to end-user, when a pubsub request is done, it uses a
+trigger to check if the requested node is or must be cached, and if possible returns
+result directly from database, otherwise it lets the normal workflow continue and query the
+pubsub service.
+
+To save resources, not all nodes are fully cached. When a node is checked, a series of
+analysers are checked, and the first one matching is used to determine if the node must be
+synchronised or not.
+
+Analysers can be registered by any plugins using ``registerAnalyser`` method:
+
+.. automethod:: sat.plugins.plugin_pubsub_cache.PubsubCache.registerAnalyser
+
+If no analyser is found, ``to_sync`` is false, or an error happens during the caching,
+the node won't be synchronised and the pubsub service will always be requested.
+
+Specifying an optional **parser** will store parsed data in addition to the raw XML of the
+items. This is more space consuming, but may be desired for the following reasons:
+
+* the parsing is resource consuming (network call or some CPU intensive operations are
+  done)
+* it is desirable to to queries on parsed data. Indeed the parsed data is stored in a
+  JSON_ field and its keys may be queried individually.
+
+The Raw XML is kept as the cache operates transparently, and a plugin may need raw data, or
+an user may be doing a low-level pubsub request.
+
+.. _JSON: https://docs.sqlalchemy.org/en/14/core/type_basics.html#sqlalchemy.types.JSON
+

doc/index.rst

@@ -27,6 +27,7 @@ You can follow this documentation to learn more on it, or join our official XMPP
    overview.rst
    configuration.rst
    components.rst
+   developer.rst
    /libervia-cli/index.rst
    /libervia-tui/index.rst
    /contributing/index.rst

doc/libervia-cli/common_arguments.rst

@@ -88,6 +88,11 @@ The common arguments for pubsub are:
   this case the ``-m, --max`` argument should be prefered. See below for RSM common
   arguments.
 
+``-C, --no-cache``
+  skip pubsub cache. By default, internal pubsub cache is used automatically if requested
+  items are available there. With this option set, a request to the pubsub service will
+  always be done, regardless of presence of node items in internal cache.
+
 result set management
 =====================
 
@@ -191,7 +196,7 @@ used, for instance, to retrieve the result formatted in JSON so the data can be
 manipulated by a script, or if you want only a specific element of the result.
 
 ``-O {…}, --output {…}``
-  specifiy the output to use. Available options depends of the command you are using,
+  specify the output to use. Available options depends of the command you are using,
   check ``li [your command] --help`` to know them.
 
   e.g.::
@@ -209,6 +214,66 @@ manipulated by a script, or if you want only a specific element of the result.
 
   Some options expect parameters, in this case they can be specified using ``=``.
 
-  e.g. specifiying a template to use::
+  e.g. specifying a template to use::
 
     $ li blog get -O template --oo browser --oo template=/tmp/my_template.html
+
+.. _time_pattern:
+
+Time Pattern
+============
+
+When a command expect a date or date with time argument, you can use a "time pattern" (you
+usually see the ``TIME_PATTERN`` as name of the argument in ``--help`` message when it can
+be used).
+
+This is a flexible way to enter a date, you can enter a date in one of the following way:
+
+- the string ``now`` to indicate current date and time;
+- an absolute date using an international format. The parser know many formats (please
+  check dateutil_ package documentation to have a detail of supported formats). Please
+  note that days are specified first and that if no time zone is specified, the local
+  time zone of your computer is assumed;
+- a relative date (or "relative delta"), see below for details on how to construct it;
+- a reference time (``now`` or absolute date as above) followed by a relative delta. If
+  the reference time is not specified, ``now`` is used;
+
+Time pattern is not case sensitive.
+
+.. _dateutil: https://dateutil.readthedocs.io
+
+Relative Delta
+--------------
+
+A relative delta is specified with:
+
+- an optional direction ``+`` for times after reference time or ``-`` for time before reference time (defaulting to ``+``);
+- a number for the quantity of units
+- a unit (e.g. seconds or minutes), see the bellow for details
+- the word ``ago`` which is same as using ``-`` for direction (direction and ``ago`` can't
+  be used at the same time)
+
+Time Units
+----------
+
+The singular or plural form of following units can be used:
+
+- ``s``, ``sec``, ``second``
+- ``m``, ``min``, ``minute``
+- ``h``, ``hr``, ``hour``
+- ``d``, ``day``
+- ``w``, ``week``
+- ``mo``, ``month``
+- ``y``, ``yr``, ``year``
+
+examples
+--------
+
+- ``2022-01-01``: first of January of 2022 at midnight
+- ``2017-02-10T13:05:00Z``: 10th of February 2017 at 13:05 UTC
+- ``2019-07-14 12:00:00 CET``: 14th of July 2019 at 12:00 CET
+- ``10 min ago``: current time minus 10 minutes
+- ``now - 10 m``: same as above (current time minus 10 minutes)
+- ``07/08/2021 +5 hours``: 7 August 2021 at midnight (local time of the computer) + 5
+  hours, i.e. 5 in the morning at local time.
+

doc/libervia-cli/debug.rst

@@ -4,7 +4,7 @@ debug: tools to help debugging/monitoring Libervia
 
 ``debug`` groups commands to monitor or manipulate Libervia and XMPP stream.
 
-.. _libervia-cli_debug_bridge_method:
+.. _li_debug_bridge_method:
 
 bridge method
 =============

doc/libervia-cli/pubsub.rst

@@ -33,6 +33,8 @@ Create an item with a custom note XML::
 
   $ echo '<note xmlns="http://example.net/mynotes">this is a note</note>' | li pubsub set -n "notes"
 
+.. _li_pubsub_get:
+
 get
 ===
 
@@ -43,15 +45,16 @@ example
 
 Retrieve the last 5 notes from our custom notes node::
 
-  $ li pubsub get -n notes -m 5
+  $ li pubsub get -n notes -M 5
 
 .. _li_pubsub_delete:
 
 delete
 ======
 
-Delete an item from a node. If ``-N, --notify`` is specified, subscribers will be notified
-of the item retraction.
+Delete an item from a node. If ``--no-notification`` is specified, subscribers wont be notified
+of the item retraction (this is NOT recommended, as it will cause trouble to keep items in
+sync, take caution when using this flag).
 
 By default a confirmation is requested before deletion is requested to the PubSub service,
 but you can override this behaviour by using ``-f, --force`` option.
@@ -330,7 +333,7 @@ returned.
 example
 -------
 
-Imagine that you want to replace all occurrences of "sàt" by "Libervia" in your personal blog. You first create a Python script like this:
+Imagine that you want to replace all occurrences of "SàT" by "Libervia" in your personal blog. You first create a Python script like this:
 
 .. sourcecode:: python
 
@@ -338,10 +341,10 @@ Imagine that you want to replace all occurrences of "sàt" by "Libervia" in your
 
    import sys
    item_raw = sys.stdin.read()
-   if not "sàt" in item_raw:
+   if not "SàT" in item_raw:
        print("SKIP")
    else:
-       print(item_raw.replace("sàt", "Libervia"))
+       print(item_raw.replace("SàT", "Libervia"))
 
 And save it a some location, e.g. ``~/expand_sat.py`` (don't forget to make it executable
 with ``chmod +x ~/expand_sat.py``).
@@ -380,3 +383,8 @@ hook
 ====
 
 Subcommands for hooks management. Please check :ref:`libervia-cli_pubsub_hook`.
+
+cache
+=====
+
+Subcommands for cache management. Please check :ref:`libervia-cli_pubsub_cache`.

doc/libervia-cli/pubsub_cache.rst

@@ -0,0 +1,350 @@
+.. _libervia-cli_pubsub_cache:
+
+=====================================
+pubsub/cache: PubSub Cache Management
+=====================================
+
+Libervia runs transparently a cache for pubsub. That means that according to internal
+criteria, some pubsub items are stored locally.
+
+The ``cache`` subcommands let user inspect and manipulate the internal cache.
+
+get
+===
+
+Retrieve items from internal cache only. Most end-users won't need to use this command, as
+the usual ``pubsub get`` command will use cache transparently. However, it may be useful
+to inspect local cache, notably for debugging.
+
+The parameters are basically the same as for :ref:`li_pubsub_get`.
+
+example
+-------
+
+Retrieve the last 2 cached items for personal blog::
+
+    $ li pubsub cache get -n urn:xmpp:microblog:0 -M 2
+
+.. _li_pubsub_cache_sync:
+
+sync
+====
+
+Synchronise or resynchronise a pubsub node. If the node is already in cache, it will be
+deleted then re-cached. Node will be put in cache even if internal policy doesn't request
+a synchronisation for this kind of nodes. Node will be (re-)subscribed to keep cache
+synchronised.
+
+All items of the node (up to the internal limit which is high), will be retrieved and put
+in cache, even if a previous version of those items have been deleted by the
+:ref:`li_pubsub_cache_purge` command.
+
+
+example
+-------
+
+Resynchronise personal blog::
+
+    $ li pubusb cache sync -n urn:xmpp:microblog:0
+
+.. _li_pubsub_cache_purge:
+
+purge
+=====
+
+Remove items from cache. This may be desirable to save resource, notably disk space.
+
+Note that once a pubsub node is cached, the cache is the source of trust. That means that
+if cache is not explicitly bypassed when retrieving items of a pubsub node (notably with
+the ``-C, --no-cache`` option of :ref:`li_pubsub_get`), only items found in cache will be
+returned, thus purged items won't be used or returned anymore even if they still exists on
+the original pubsub service.
+
+If you have purged items by mistake, it is possible to retrieve them either node by node
+using :ref:`li_pubsub_cache_sync`, or by resetting the whole pubsub cache with
+:ref:`li_pubsub_cache_reset`.
+
+If you have a node or a profile (e.g. a component) caching a lot of items frequently, you
+may use this command using a scheduler like cron_.
+
+.. _cron: https://en.wikipedia.org/wiki/Cron
+
+examples
+--------
+
+Remove all blog and event items from cache if they haven't been updated since 6 months::
+
+    $ li pubsub cache purge -t blog -t event -b "6 months ago"
+
+Remove items from profile ``ap_gateway`` if they have been created more that 2 months
+ago::
+
+    $ li pubsub cache purge -p ap_gateway --created-before "2 months ago"
+
+.. _li_pubsub_cache_reset:
+
+reset
+=====
+
+Reset the whole pubsub cache. This means that all nodes and all them items will be removed
+from cache. After this command, cache will be re-filled progressively as if it where a new
+one.
+
+.. note::
+
+    Use this command with caution: even if cache will be re-constructed with time, that
+    means that items will have to be retrieved again, that may be resource intensive both
+    for your machine and for the pubsub services which will be used. That also means that
+    searching items will return less results until all desired items are cached again.
+
+    Also note that all items of cached nodes are retrieved, even if you have previously
+    purged items, they will be retrieved again.
+
+example
+-------
+
+Reset the whole pubsub cache::
+
+    $ li pubsub cache reset
+
+search
+======
+
+Search items into pubsub cache. The search is done on the whole cache, it's not restricted
+to a single node/profile (even if it may be if suitable filters are specified). Full-Text
+Search can be done with ``-f FTS, --fts FTS`` argument, as well as filtering on parsed
+data (with ``-F PATH OPERATOR VALUE, --field PATH OPERATOR VALUE``, see below).
+
+By default, parsed data are returned, with the 3 additional keys ``pubsub_service``,
+``pubsub_items`` (the search being done on the whole cache, those data are here to get the
+full location of each item) and ``node_profile``.
+
+"Parsed data" are the result of the parsing of the items XML payload by feature aware
+plugins. Those data are usually more readable and easier to work with. Parsed data are
+only stored when a parser is registered for a specific feature, that means that a Pubsub
+item in cache may not have parsed data at all, in which case an empty dict will be used
+instead (and ``-P, --payload`` argument should be used to get content of the item).
+
+The dates are normally stored as `Unix time`_ in database, but the default output convert
+the ``updated``, ``created`` and ``published`` fields to human readable local time. Use
+``--output simple`` if you want to keep the float (or int) value.
+
+XML item payload is not returned by default, but it can be added to the ``item_payload``
+field if ``-P, --payload`` argument is set. You can also use the ``--output xml`` (or
+``xml_raw`` if you don't want prettifying) to output directly the highlighted XML
+— without the parsed data —, to have an output similar to the one of ``li pubsub get``.
+
+If you are interested only in a specific data (e.g. item id and title), the ``-k KEY,
+--key KEY`` can be used.
+
+You'll probably want to limit result size by using ``-l LIMIT, --limit LIMIT``, and do
+pagination using ``-i INDEX, --index INDEX``.
+
+.. _Unix time: https://en.wikipedia.org/wiki/Unix_time
+
+Filters
+-------
+
+By default search returns all items in cache, you have to use filter to specify what you
+are looking after. We can split filters in 3 categories: nodes/items metadata,
+Full-Text Search query and parsed metadata.
+
+Nodes/items metadata are the generic information you have on a node: which profile it
+belong too, which pubsub service it's coming from, what's the name or type of the node,
+etc.
+
+Arguments there should be self-explanatory. Type (set with ``-t TYPE, --type TYPE``) and
+subtype (set with ``-S SUBTYPE, --subtype SUBTYPE``) are values dependent of the
+plugin/feature associated with the node, so we can't list them in an exhaustive way here.
+The most common type is probably ``blog``, from which a subtype can be ``comment``. An
+empty string can be used to find items with (sub)type not set.
+
+It's usually a good idea to specify a profile with ``-p PROFILE, --profile PROFILE``,
+otherwise you may get duplicated results.
+
+Full-Text Search
+----------------
+
+You can specify a Full-Text Search query with the ``-f FTS_QUERY, --fts FTS_QUERY``
+argument. The engine is currently SQLite FTS5, and you can check its `query syntax`_.
+FTS is done on the whole raw XML payload, that means that all data there can be matched
+(including XML tags and attributes).
+
+FTS queries are indexed, that means that they are fast and efficient.
+
+.. note::
+
+  Futures version of Libervia will probably include other FTS engines (support for
+  PostgreSQL and MySQL/MariaDB is planned). Thus the syntax may vary depending on the
+  engine, or a common syntax may be implemented for all engines in the future. Keep that
+  in mind if you plan to use FTS capabilities in long-term queries, e.g. in scripts.
+
+.. _query syntax: https://sqlite.org/fts5.html#full_text_query_syntax
+
+Parsed Metadata Filters
+-----------------------
+
+It is possible to filter on any field of parsed data. This is done with the ``-F PATH
+OPERATOR VALUE, --field PATH OPERATOR VALUE`` (be careful that the short option is an
+uppercase ``F``, the lower case one being used for Full-Text Search).
+
+.. note::
+
+  Parsed Metadata Filters are not indexed, that means that using them is less efficient
+  than using e.g. Full-Text Search. If you want to filter on a text field, it's often a
+  good idea to pre-filter using Full-Text Search to have more efficient queries.
+
+``PATH`` and ``VALUE`` can be either specified as string, or using JSON syntax (if the
+value can't be decoded as JSON, it is used as plain text).
+
+``PATH`` is the name of the field to use. If you must go beyond root level fields, you can
+use a JSON array to specify each element of the path. If a string is used, it's an object
+key, if a number is used it's an array index. Thus you can use ``title`` to access the
+root title key, or ``'"title"'`` (JSON string escaped for shell) or ``'["title"]'`` (JSON
+array with the "title" string, escaped for shell).
+
+.. note::
+
+  The extra fields ``pubsub_service``, ``pubsub_node`` and  ``node_profile`` are added to
+  the result after the query, thus they can't be used as fields for filtering (use the
+  direct arguments for that).
+
+``OPERATOR`` indicate how to use the value to make a filter. The currently supported
+operators are:
+
+``==`` or ``eq``
+  Equality operator, true if field value is the same as given value.
+
+``!=`` or ``ne``
+  Inequality operator, true if the field value is different from given value.
+
+``>`` or ``gt``
+  Greater than, true if the field value is higher than given value. For string, this is
+  according to alphabetical order.
+
+  Time Pattern can be used here, see below.
+
+``<`` or ``lt``
+  Lesser than, true if the field value is lower than given value. For string, this is
+  according to alphabetical order.
+
+  Time Pattern can be used here, see below.
+
+``between``
+  Given value must be an array with 2 elements. The condition is true if field value is
+  between the 2 elements (for string, this is according to alphabetical order).
+
+  Time Pattern can be used here, see below.
+
+``in``
+  Given value must be an array of elements. Field value must be one of them to make the
+  condition true.
+
+``not_in``
+  Given value must be an array of elements. Field value must not be any of them the make
+  the condition true.
+
+``overlap``
+  This can be used only on array fields.
+
+  If given value is not already an array, it is put in an array. Condition is true if any
+  element of field value match any element of given value. Notably useful to filter on
+  tags.
+
+``ioverlap``
+  Same as ``overlap`` but done in a case insensitive way.
+
+``disjoint``
+  This can be used only on array fields.
+
+  If given value is not already an array, it is put in an array. Condition is true if no
+  element of field value match any element of given value. Notably useful to filter out
+  tags.
+
+``idisjoint``
+  Same as ``disjoint`` but done in a case insensitive way.
+
+``like``
+  Does pattern matching on a string. ``%`` can be used to match zero or more characters
+  and ``_`` can be used to match any single character.
+
+  If you're not looking after a specific field, it's better to use Full-Text Search when
+  possible.
+
+``ilike``
+  Like ``like`` but done in a case insensitive way.
+
+
+``not_like``
+  Same as ``like`` except that condition is true when pattern is **not** matching.
+
+``not_ilike``
+  Same as ``not_like`` but done in a case insensitive way.
+
+
+For ``gt``/``>``, ``lt``/``<`` and ``between``, you can use :ref:`time_pattern` by using
+the syntax ``TP(<time pattern>)`` (see examples below).
+
+Ordering
+--------
+
+Result ordering can be done by a well know order, or using a parsed data field. Ordering
+default to ``created`` (see below), but this may be changed with ``-o ORDER [FIELD]
+[DIRECTION], --order-by ORDER [FIELD] [DIRECTION]``.
+
+``ORDER`` can be one of the following:
+
+``creation``
+  Order by item creation date. Note that is this the date of creation of the item in cache
+  (which most of time should correspond to order of creation of the item in the source
+  pubsub service), and this may differ from the date of publication as specified with some
+  feature (like blog). This is important when old items are imported, e.g. when they're
+  coming from an other blog engine.
+
+``modification``
+  Order by the date when item has last been modified. Modification date is the same as
+  creation date if the item has never been modified since it is in cache. The same warning
+  as for ``creation`` applies: this is the date of last modification in cache, not the one
+  advertised in parsed data.
+
+``item_id``
+  Order by XMPP id of the item. Notably useful when user-friendly ID are used (like it is
+  often the case with blogs).
+
+``rank``
+  Order item by Full-Text Search rank. This one can only be used when Full-Text Search is
+  used (via ``-f FTS_QUERY, --fts FTS_QUERY``). Rank is a value indicating how well an
+  item match the query. This usually needs to be used with ``desc`` direction, so you get
+  most relevant items first.
+
+``field``
+  This special order indicates that the ordering must be done on an parsed data field. The
+  following argument is then the path of the field to used (which can be a plain text name
+  of a root field, or a JSON encoded array). An optional direction can be specified as a
+  third argument. See examples below.
+
+examples
+--------
+
+Search for blog items cached for the profile ``louise`` which contain the word
+``Slovakia``::
+
+  $ li pubsub cache search -t blog -p louise -f Slovakia
+
+Show title, publication date and id of blog articles (excluding comments) which have been
+published on Louise's blog during the last 6 months, order them by item id. Here we use an
+empty string as a subtype to exclude comments (for which subtype is ``comment``)::
+
+  $ li pubsub cache search -t blog -S "" -p louise -s louise@example.net -n urn:xmpp:microblog:0 -F published gt 'TP(6 months ago)' -k id -k published -k title -o item_id
+
+Show all blog items from anywhere which are tagged as XMPP or ActivityPub (case
+insensitive) and which have been published in the last month (according to advertised
+publishing date, not cache creation date).
+
+We want to order them by descending publication date (again the advertised publication
+date, not cache creation), and we don't want more than 50 results.
+
+We do a FTS query there even if it's not mandatory, because it will do an efficient
+pre-filtering::
+
+  $ li pubsub cache search -f "xmpp OR activitypub" -F tags ioverlap '["xmpp", "activitypub"]' -F published gt 'TP(1 month ago)' -o field published desc -l 50

doc/libervia-cli/pubsub_hook.rst

@@ -4,7 +4,7 @@
 pubsub/hook: PubSub hooks management
 ====================================
 
-``hook`` is a subcommands grouping all PubSub commands related to hooks management. Hooks
+``hook`` is a subcommand grouping all PubSub commands related to hooks management. Hooks
 are user actions launched on specific events.
 
 3 types of hooks can be used:

docker/backend-dev-e2e/Dockerfile

@@ -46,7 +46,7 @@ USER libervia
 
 RUN \
     # pytest and its plugins
-    pip install pytest pytest-timeout pytest-dependency \
+    pip install pytest pytest-timeout pytest-dependency pytest-twisted \
     # needed to test libervia-cli
     sh \
     # needed to test libervia

sat/VERSION

@@ -1 +1 @@
-0.8.0
+0.9.0D

sat/bridge/bridge_constructor/base_constructor.py

@@ -49,7 +49,7 @@ class Constructor(object):
     FRONTEND_TEMPLATE = None
     FRONTEND_DEST = None
 
-    # set to False if your bridge need only core
+    # set to False if your bridge needs only core
     FRONTEND_ACTIVATE = True
 
     def __init__(self, bridge_template, options):
@@ -284,6 +284,7 @@ class Constructor(object):
                 "args": self.getArguments(
                     function["sig_in"], name=arg_doc, default=default
                 ),
+                "args_no_default": self.getArguments(function["sig_in"], name=arg_doc),
             }
 
             extend_method = getattr(

sat/bridge/bridge_constructor/constructors/dbus/constructor.py

@@ -25,20 +25,19 @@ class DbusConstructor(base_constructor.Constructor):
     CORE_TEMPLATE = "dbus_core_template.py"
     CORE_DEST = "dbus_bridge.py"
     CORE_FORMATS = {
-        "signals": """\
-    @dbus.service.signal(const_INT_PREFIX+const_{category}_SUFFIX,
-                         signature='{sig_in}')
-    def {name}(self, {args}):
-        {body}\n""",
+        "methods_declarations": """\
+        Method('{name}', arguments='{sig_in}', returns='{sig_out}'),""",
+
         "methods": """\
-    @dbus.service.method(const_INT_PREFIX+const_{category}_SUFFIX,
-                         in_signature='{sig_in}', out_signature='{sig_out}',
-                         async_callbacks={async_callbacks})
-    def {name}(self, {args}{async_comma}{async_args_def}):
-        {debug}return self._callback("{name}", {args_result}{async_comma}{async_args_call})\n""",
-        "signal_direct_calls": """\
+    def dbus_{name}(self, {args}):
+        {debug}return self._callback("{name}", {args_no_default})\n""",
+
+        "signals_declarations": """\
+        Signal('{name}', '{sig_in}'),""",
+
+        "signals": """\
     def {name}(self, {args}):
-        self.dbus_bridge.{name}({args})\n""",
+        self._obj.emitSignal("{name}", {args})\n""",
     }
 
     FRONTEND_TEMPLATE = "dbus_frontend_template.py"
@@ -68,17 +67,10 @@ class DbusConstructor(base_constructor.Constructor):
     def core_completion_method(self, completion, function, default, arg_doc, async_):
         completion.update(
             {
-                "debug": ""
-                if not self.args.debug
-                else 'log.debug ("%s")\n%s' % (completion["name"], 8 * " "),
-                "args_result": self.getArguments(
-                    function["sig_in"], name=arg_doc, unicode_protect=self.args.unicode
-                ),
-                "async_comma": ", " if async_ and function["sig_in"] else "",
-                "async_args_def": "callback=None, errback=None" if async_ else "",
-                "async_args_call": "callback=callback, errback=errback" if async_ else "",
-                "async_callbacks": "('callback', 'errback')" if async_ else "None",
-                "category": completion["category"].upper(),
+                "debug": (
+                    "" if not self.args.debug
+                    else f'log.debug ("{completion["name"]}")\n{8 * " "}'
+                )
             }
         )
 

sat/bridge/bridge_constructor/constructors/dbus/dbus_core_template.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# SàT communication bridge
+# Libervia communication bridge
 # Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
 
 # This program is free software: you can redistribute it and/or modify
@@ -16,15 +16,15 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from types import MethodType
+from functools import partialmethod
+from twisted.internet import defer, reactor
 from sat.core.i18n import _
-import dbus
-import dbus.service
-import dbus.mainloop.glib
-import inspect
 from sat.core.log import getLogger
-from sat.tools import config
-from twisted.internet.defer import Deferred
 from sat.core.exceptions import BridgeInitError
+from sat.tools import config
+from txdbus import client, objects, error
+from txdbus.interface import DBusInterface, Method, Signal
 
 
 log = getLogger(__name__)
@@ -45,251 +45,127 @@ class ParseError(Exception):
     pass
 
 
-class MethodNotRegistered(dbus.DBusException):
-    _dbus_error_name = const_ERROR_PREFIX + ".MethodNotRegistered"
+class DBusException(Exception):
+    pass
 
 
-class InternalError(dbus.DBusException):
-    _dbus_error_name = const_ERROR_PREFIX + ".InternalError"
+class MethodNotRegistered(DBusException):
+    dbusErrorName = const_ERROR_PREFIX + ".MethodNotRegistered"
 
 
-class AsyncNotDeferred(dbus.DBusException):
-    _dbus_error_name = const_ERROR_PREFIX + ".AsyncNotDeferred"
-
-
-class DeferredNotAsync(dbus.DBusException):
-    _dbus_error_name = const_ERROR_PREFIX + ".DeferredNotAsync"
-
-
-class GenericException(dbus.DBusException):
+class GenericException(DBusException):
     def __init__(self, twisted_error):
         """
 
         @param twisted_error (Failure): instance of twisted Failure
-        @return: DBusException
+        error message is used to store a repr of message and condition in a tuple,
+        so it can be evaluated by the frontend bridge.
         """
-        super(GenericException, self).__init__()
         try:
             # twisted_error.value is a class
             class_ = twisted_error.value().__class__
         except TypeError:
             # twisted_error.value is an instance
             class_ = twisted_error.value.__class__
-            message = twisted_error.getErrorMessage()
+            data = twisted_error.getErrorMessage()
             try:
-                self.args = (message, twisted_error.value.condition)
+                data = (data, twisted_error.value.condition)
             except AttributeError:
-                self.args = (message,)
-        self._dbus_error_name = ".".join(
-            [const_ERROR_PREFIX, class_.__module__, class_.__name__]
+                data = (data,)
+        else:
+            data = (str(twisted_error),)
+        self.dbusErrorName = ".".join(
+            (const_ERROR_PREFIX, class_.__module__, class_.__name__)
         )
+        super(GenericException, self).__init__(repr(data))
+
+    @classmethod
+    def create_and_raise(cls, exc):
+        raise cls(exc)
 
 
-class DbusObject(dbus.service.Object):
-    def __init__(self, bus, path):
-        dbus.service.Object.__init__(self, bus, path)
-        log.debug("Init DbusObject...")
+class DBusObject(objects.DBusObject):
+
+    core_iface = DBusInterface(
+        const_INT_PREFIX + const_CORE_SUFFIX,
+##METHODS_DECLARATIONS_PART##
+##SIGNALS_DECLARATIONS_PART##
+    )
+    plugin_iface = DBusInterface(
+        const_INT_PREFIX + const_PLUGIN_SUFFIX
+    )
+
+    dbusInterfaces = [core_iface, plugin_iface]
+
+    def __init__(self, path):
+        super().__init__(path)
+        log.debug("Init DBusObject...")
         self.cb = {}
 
     def register_method(self, name, cb):
         self.cb[name] = cb
 
     def _callback(self, name, *args, **kwargs):
-        """call the callback if it exists, raise an exception else
-        if the callback return a deferred, use async methods"""
-        if not name in self.cb:
+        """Call the callback if it exists, raise an exception else"""
+        try:
+            cb = self.cb[name]
+        except KeyError:
             raise MethodNotRegistered
-
-        if "callback" in kwargs:
-            # we must have errback too
-            if not "errback" in kwargs:
-                log.error("errback is missing in method call [%s]" % name)
-                raise InternalError
-            callback = kwargs.pop("callback")
-            errback = kwargs.pop("errback")
-            async_ = True
         else:
-            async_ = False
-        result = self.cb[name](*args, **kwargs)
-        if async_:
-            if not isinstance(result, Deferred):
-                log.error("Asynchronous method [%s] does not return a Deferred." % name)
-                raise AsyncNotDeferred
-            result.addCallback(
-                lambda result: callback() if result is None else callback(result)
-            )
-            result.addErrback(lambda err: errback(GenericException(err)))
-        else:
-            if isinstance(result, Deferred):
-                log.error("Synchronous method [%s] return a Deferred." % name)
-                raise DeferredNotAsync
-            return result
-
-    ### signals ###
-
-    @dbus.service.signal(const_INT_PREFIX + const_PLUGIN_SUFFIX, signature="")
-    def dummySignal(self):
-        # FIXME: workaround for addSignal (doesn't work if one method doensn't
-        #       already exist for plugins), probably missing some initialisation, need
-        #       further investigations
-        pass
-
-##SIGNALS_PART##
-    ### methods ###
+            d = defer.maybeDeferred(cb, *args, **kwargs)
+            d.addErrback(GenericException.create_and_raise)
+            return d
 
 ##METHODS_PART##
-    def __attributes(self, in_sign):
-        """Return arguments to user given a in_sign
-        @param in_sign: in_sign in the short form (using s,a,i,b etc)
-        @return: list of arguments that correspond to a in_sign (e.g.: "sss" return "arg1, arg2, arg3")"""
-        i = 0
-        idx = 0
-        attr = []
-        while i < len(in_sign):
-            if in_sign[i] not in ["b", "y", "n", "i", "x", "q", "u", "t", "d", "s", "a"]:
-                raise ParseError("Unmanaged attribute type [%c]" % in_sign[i])
 
-            attr.append("arg_%i" % idx)
-            idx += 1
+class Bridge:
 
-            if in_sign[i] == "a":
-                i += 1
-                if (
-                    in_sign[i] != "{" and in_sign[i] != "("
-                ):  # FIXME: must manage tuples out of arrays
-                    i += 1
-                    continue  # we have a simple type for the array
-                opening_car = in_sign[i]
-                assert opening_car in ["{", "("]
-                closing_car = "}" if opening_car == "{" else ")"
-                opening_count = 1
-                while True:  # we have a dict or a list of tuples
-                    i += 1
-                    if i >= len(in_sign):
-                        raise ParseError("missing }")
-                    if in_sign[i] == opening_car:
-                        opening_count += 1
-                    if in_sign[i] == closing_car:
-                        opening_count -= 1
-                        if opening_count == 0:
-                            break
-            i += 1
-        return attr
-
-    def addMethod(self, name, int_suffix, in_sign, out_sign, method, async_=False):
-        """Dynamically add a method to Dbus Bridge"""
-        inspect_args = inspect.getfullargspec(method)
-
-        _arguments = inspect_args.args
-        _defaults = list(inspect_args.defaults or [])
-
-        if inspect.ismethod(method):
-            # if we have a method, we don't want the first argument (usually 'self')
-            del (_arguments[0])
-
-        # first arguments are for the _callback method
-        arguments_callback = ", ".join(
-            [repr(name)]
-            + (
-                (_arguments + ["callback=callback", "errback=errback"])
-                if async_
-                else _arguments
-            )
-        )
-
-        if async_:
-            _arguments.extend(["callback", "errback"])
-            _defaults.extend([None, None])
-
-        # now we create a second list with default values
-        for i in range(1, len(_defaults) + 1):
-            _arguments[-i] = "%s = %s" % (_arguments[-i], repr(_defaults[-i]))
-
-        arguments_defaults = ", ".join(_arguments)
-
-        code = compile(
-            "def %(name)s (self,%(arguments_defaults)s): return self._callback(%(arguments_callback)s)"
-            % {
-                "name": name,
-                "arguments_defaults": arguments_defaults,
-                "arguments_callback": arguments_callback,
-            },
-            "<DBus bridge>",
-            "exec",
-        )
-        exec(code)  # FIXME: to the same thing in a cleaner way, without compile/exec
-        method = locals()[name]
-        async_callbacks = ("callback", "errback") if async_ else None
-        setattr(
-            DbusObject,
-            name,
-            dbus.service.method(
-                const_INT_PREFIX + int_suffix,
-                in_signature=in_sign,
-                out_signature=out_sign,
-                async_callbacks=async_callbacks,
-            )(method),
-        )
-        function = getattr(self, name)
-        func_table = self._dbus_class_table[
-            self.__class__.__module__ + "." + self.__class__.__name__
-        ][function._dbus_interface]
-        func_table[function.__name__] = function  # Needed for introspection
-
-    def addSignal(self, name, int_suffix, signature, doc={}):
-        """Dynamically add a signal to Dbus Bridge"""
-        attributes = ", ".join(self.__attributes(signature))
-        # TODO: use doc parameter to name attributes
-
-        # code = compile ('def '+name+' (self,'+attributes+'): log.debug ("'+name+' signal")', '<DBus bridge>','exec') #XXX: the log.debug is too annoying with xmllog
-        code = compile(
-            "def " + name + " (self," + attributes + "): pass", "<DBus bridge>", "exec"
-        )
-        exec(code)
-        signal = locals()[name]
-        setattr(
-            DbusObject,
-            name,
-            dbus.service.signal(const_INT_PREFIX + int_suffix, signature=signature)(
-                signal
-            ),
-        )
-        function = getattr(self, name)
-        func_table = self._dbus_class_table[
-            self.__class__.__module__ + "." + self.__class__.__name__
-        ][function._dbus_interface]
-        func_table[function.__name__] = function  # Needed for introspection
-
-
-class Bridge(object):
     def __init__(self):
-        dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
         log.info("Init DBus...")
+        self._obj = DBusObject(const_OBJ_PATH)
+
+    async def postInit(self):
         try:
-            self.session_bus = dbus.SessionBus()
-        except dbus.DBusException as e:
-            if e._dbus_error_name == "org.freedesktop.DBus.Error.NotSupported":
+            conn = await client.connect(reactor)
+        except error.DBusException as e:
+            if e.errName == "org.freedesktop.DBus.Error.NotSupported":
                 log.error(
                     _(
-                        "D-Bus is not launched, please see README to see instructions on how to launch it"
+                        "D-Bus is not launched, please see README to see instructions on "
+                        "how to launch it"
                     )
                 )
-            raise BridgeInitError
-        self.dbus_name = dbus.service.BusName(const_INT_PREFIX, self.session_bus)
-        self.dbus_bridge = DbusObject(self.session_bus, const_OBJ_PATH)
+            raise BridgeInitError(str(e))
 
-##SIGNAL_DIRECT_CALLS_PART##
+        conn.exportObject(self._obj)
+        await conn.requestBusName(const_INT_PREFIX)
+
+##SIGNALS_PART##
     def register_method(self, name, callback):
-        log.debug("registering DBus bridge method [%s]" % name)
-        self.dbus_bridge.register_method(name, callback)
+        log.debug(f"registering DBus bridge method [{name}]")
+        self._obj.register_method(name, callback)
 
-    def addMethod(self, name, int_suffix, in_sign, out_sign, method, async_=False, doc={}):
-        """Dynamically add a method to Dbus Bridge"""
+    def emitSignal(self, name, *args):
+        self._obj.emitSignal(name, *args)
+
+    def addMethod(
+            self, name, int_suffix, in_sign, out_sign, method, async_=False, doc={}
+    ):
+        """Dynamically add a method to D-Bus Bridge"""
         # FIXME: doc parameter is kept only temporary, the time to remove it from calls
-        log.debug("Adding method [%s] to DBus bridge" % name)
-        self.dbus_bridge.addMethod(name, int_suffix, in_sign, out_sign, method, async_)
+        log.debug(f"Adding method {name!r} to D-Bus bridge")
+        self._obj.plugin_iface.addMethod(
+            Method(name, arguments=in_sign, returns=out_sign)
+        )
+        # we have to create a method here instead of using partialmethod, because txdbus
+        # uses __func__ which doesn't work with partialmethod
+        def caller(self_, *args, **kwargs):
+            return self_._callback(name, *args, **kwargs)
+        setattr(self._obj, f"dbus_{name}", MethodType(caller, self._obj))
         self.register_method(name, method)
 
     def addSignal(self, name, int_suffix, signature, doc={}):
-        self.dbus_bridge.addSignal(name, int_suffix, signature, doc)
-        setattr(Bridge, name, getattr(self.dbus_bridge, name))
+        """Dynamically add a signal to D-Bus Bridge"""
+        log.debug(f"Adding signal {name!r} to D-Bus bridge")
+        self._obj.plugin_iface.addSignal(Signal(name, signature))
+        setattr(Bridge, name, partialmethod(Bridge.emitSignal, name))

sat/core/constants.py

@@ -216,6 +216,7 @@ class Const(object):
     PLUG_TYPE_EXP = "EXP"
     PLUG_TYPE_SEC = "SEC"
     PLUG_TYPE_SYNTAXE = "SYNTAXE"
+    PLUG_TYPE_PUBSUB = "PUBSUB"
     PLUG_TYPE_BLOG = "BLOG"
     PLUG_TYPE_IMPORT = "IMPORT"
     PLUG_TYPE_ENTRY_POINT = "ENTRY_POINT"
@@ -237,9 +238,10 @@ class Const(object):
     PS_PUBLISH = "publish"
     PS_RETRACT = "retract"  # used for items
     PS_DELETE = "delete"  # used for nodes
+    PS_PURGE = "purge"  # used for nodes
     PS_ITEM = "item"
     PS_ITEMS = "items"  # Can contain publish and retract items
-    PS_EVENTS = (PS_ITEMS, PS_DELETE)
+    PS_EVENTS = (PS_ITEMS, PS_DELETE, PS_PURGE)
 
     ## MESSAGE/NOTIFICATION LEVELS ##
 
@@ -366,6 +368,7 @@ class Const(object):
 
     ## Common extra keys/values ##
     KEY_ORDER_BY = "order_by"
+    KEY_USE_CACHE = "use_cache"
 
     ORDER_BY_CREATION = 'creation'
     ORDER_BY_MODIFICATION = 'modification'