erebion/libervia-backend
Watch
1
0
Fork
You've already forked libervia-backend
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

plugins XEP-0059, XEP-0060, XEP-0277, XEP-0313: better serialisation:

Browse source 
`data_format.serialise` is now used for `mbGet`, and RSM/MAM values are not transtyped to
strings anymore. A serialised dict is now used, items are put in the `items` key.

Comments handling has been refactored to use a list for the potentially multiple comments
nodes.

`rsm` data are now in a `rsm` key of the dict, and `mam` data are merged with other
metadata.
This commit is contained in:
Goffi 2020-07-16 09:07:20 +02:00
commit e03eb5bfcc
5 changed files with 153 additions and 166 deletions
Download patch file Download diff file Expand all files Collapse all files

14
sat/plugins/plugin_xep_0059.py
View file

@ -77,8 +77,8 @@ class XEP_0059(object):
else:
return None
def serialise(self, rsm_response, data=None):
"""Serialise data for RSM
def response2dict(self, rsm_response, data=None):
"""Return a dict with RSM response
Key set in data can be:
- rsm_first: first item id in the page
@ -87,7 +87,7 @@ class XEP_0059(object):
- rsm_count: total number of items in the full set (may be approximage)
If a value doesn't exists, it's not set.
All values are set as strings.
@param rsm_response(rsm.RSMResponse): response to serialise
@param rsm_response(rsm.RSMResponse): response to parse
@param data(dict, None): dict to update with rsm_* data.
If None, a new dict is created
@return (dict): data dict
@ -95,13 +95,11 @@ class XEP_0059(object):
if data is None:
data = {}
if rsm_response.first is not None:
data["rsm_first"] = rsm_response.first
data["first"] = rsm_response.first
if rsm_response.last is not None:
data["rsm_last"] = rsm_response.last
data["last"] = rsm_response.last
if rsm_response.index is not None:
data["rsm_index"] = str(rsm_response.index)
if rsm_response.index is not None:
data["rsm_index"] = str(rsm_response.index)
data["index"] = rsm_response.index
return data

73
sat/plugins/plugin_xep_0060.py
View file

@ -717,16 +717,24 @@ class XEP_0060(object):
"node": node,
"uri": self.getNodeURI(service_jid, node),
}
if rsm_request is not None and rsm_response is not None:
metadata.update(
{
"rsm_" + key: value
for key, value in rsm_response.toDict().items()
}
)
if mam_response is not None:
for key, value in mam_response.items():
metadata["mam_" + key] = value
# mam_response is a dict with "complete" and "stable" keys
# we can put them directly in metadata
metadata.update(mam_response)
if rsm_request is not None and rsm_response is not None:
metadata['rsm'] = rsm_response.toDict()
if mam_response is None:
index = rsm_response.index
count = rsm_response.count
if index is None or count is None:
# we don't have enough information to know if the data is complete
# or not
metadata["complete"] = None
else:
# normally we have a strict equality here but XEP-0059 states
# that index MAY be approximative, so just in case…
metadata["complete"] = index + len(items) >= count
return (items, metadata)
d.addCallback(addMetadata)
@ -1107,6 +1115,7 @@ class XEP_0060(object):
@param node(unicode): node
@return (unicode): URI of the node
"""
# FIXME: deprecated, use sat.tools.common.uri instead
assert service is not None
# XXX: urllib.urlencode use "&" to separate value, while XMPP URL (cf. RFC 5122)
# use ";" as a separator. So if more than one value is used in query_data,
@ -1127,66 +1136,40 @@ class XEP_0060(object):
):
return self.rt_sessions.getResults(session_id, on_success, on_error, profile)
def transItemsData(self, items_data, item_cb=lambda item: item.toXml(),
serialise=False):
def transItemsData(self, items_data, item_cb=lambda item: item.toXml()):
"""Helper method to transform result from [getItems]
the items_data must be a tuple(list[domish.Element], dict[unicode, unicode])
as returned by [getItems]. metadata values are then casted to unicode and
each item is passed to items_cb then optionally serialised with
data_format.serialise.
as returned by [getItems].
@param items_data(tuple): tuple returned by [getItems]
@param item_cb(callable): method to transform each item
@param serialise(bool): if True do a data_format.serialise
after applying item_cb
@return (tuple): a serialised form ready to go throught bridge
"""
items, metadata = items_data
if serialise:
items = [data_format.serialise(item_cb(item)) for item in items]
else:
items = [item_cb(item) for item in items]
items = [item_cb(item) for item in items]
return (
items,
{key: str(value) for key, value in metadata.items()},
)
return (items, metadata)
def transItemsDataD(self, items_data, item_cb, serialise=False):
def transItemsDataD(self, items_data, item_cb):
"""Helper method to transform result from [getItems], deferred version
the items_data must be a tuple(list[domish.Element], dict[unicode, unicode])
as returned by [getItems]. metadata values are then casted to unicode and
each item is passed to items_cb then optionally serialised with
data_format.serialise.
each item is passed to items_cb.
An errback is added to item_cb, and when it is fired the value is filtered from
final items
@param items_data(tuple): tuple returned by [getItems]
@param item_cb(callable): method to transform each item (must return a deferred)
@param serialise(bool): if True do a data_format.serialise
after applying item_cb
@return (tuple): a deferred which fire a serialised form ready to go throught
bridge
@return (tuple): a deferred which fire a dict which can be serialised to go
throught bridge
"""
items, metadata = items_data
def eb(failure_):
log.warning(f"Error while serialising/parsing item: {failure_.value}")
log.warning(f"Error while parsing item: {failure_.value}")
d = defer.gatherResults([item_cb(item).addErrback(eb) for item in items])
def finishSerialisation(parsed_items):
if serialise:
items = [data_format.serialise(i) for i in parsed_items if i is not None]
else:
items = [i for i in parsed_items if i is not None]
return (
items,
{key: str(value) for key, value in metadata.items()},
)
d.addCallback(finishSerialisation)
d.addCallback(lambda parsed_items: (parsed_items, metadata))
return d
def serDList(self, results, failure_result=None):

158
sat/plugins/plugin_xep_0277.py
View file

@ -103,7 +103,7 @@ class XEP_0277(object):
"mbGet",
".plugin",
in_sign="ssiasa{ss}s",
out_sign="(asa{ss})",
out_sign="s",
method=self._mbGet,
async_=True,
)
@ -357,21 +357,25 @@ class XEP_0277(object):
)
# links
comments = microblog_data['comments'] = []
for link_elt in entry_elt.elements(NS_ATOM, "link"):
if (
link_elt.getAttribute("rel") == "replies"
and link_elt.getAttribute("title") == "comments"
):
key = check_conflict("comments", True)
microblog_data[key] = link_elt["href"]
uri = link_elt["href"]
comments_data = {
"uri": uri,
}
try:
service, node = self.parseCommentUrl(microblog_data[key])
service, node = self.parseCommentUrl(uri)
except Exception as e:
log.warning(f"Can't parse url {microblog_data[key]}: {e}")
del microblog_data[key]
log.warning(f"Can't parse comments url: {e}")
continue
else:
microblog_data["{}_service".format(key)] = service.full()
microblog_data["{}_node".format(key)] = node
comments_data["service"] = service.full()
comments_data["node"] = node
comments.append(comments_data)
else:
rel = link_elt.getAttribute("rel", "")
title = link_elt.getAttribute("title", "")
@ -589,9 +593,10 @@ class XEP_0277(object):
entry_elt.addElement("id", content=entry_id) #
## comments ##
if "comments" in data:
for comments_data in data.get('comments', []):
link_elt = entry_elt.addElement("link")
link_elt["href"] = data["comments"]
# XXX: "uri" is set in self._manageComments if not already existing
link_elt["href"] = comments_data["uri"]
link_elt["rel"] = "replies"
link_elt["title"] = "comments"
@ -646,8 +651,8 @@ class XEP_0277(object):
def _manageComments(self, client, mb_data, service, node, item_id, access=None):
"""Check comments keys in mb_data and create comments node if necessary
if mb_data['comments'] exists, it is used (or mb_data['comments_service'] and/or mb_data['comments_node']),
else it is generated (if allow_comments is True).
if a comments node metadata is set in the mb_data['comments'] list, it is used
otherwise it is generated (if allow_comments is True).
@param mb_data(dict): microblog mb_data
@param service(jid.JID, None): PubSub service of the parent item
@param node(unicode): node of the parent item
@ -655,11 +660,13 @@ class XEP_0277(object):
@param access(unicode, None): access model
None to use same access model as parent item
"""
# FIXME: if 'comments' already exists in mb_data,
# it is not used to create the Node
allow_comments = mb_data.pop("allow_comments", None)
if allow_comments is None:
return
if "comments" in mb_data:
mb_data["allow_comments"] = True
else:
# no comments set or requested, nothing to do
return
elif allow_comments == False:
if "comments" in mb_data:
log.warning(
@ -671,6 +678,13 @@ class XEP_0277(object):
del mb_data["comments"]
return
# we have usually a single comment node, but the spec allow several, so we need to
# handle this in a list
if len(mb_data.setdefault('comments', [])) == 0:
# we need at least one comment node
comments_data = {}
mb_data['comments'].append({})
if access is None:
# TODO: cache access models per service/node
parent_node_config = yield self._p.getConfiguration(client, service, node)
@ -689,63 +703,65 @@ class XEP_0277(object):
# if other plugins need to change the options
yield self.host.trigger.point("XEP-0277_comments", client, mb_data, options)
try:
comments_node = mb_data["comments_node"]
except KeyError:
comments_node = self.getCommentsNode(item_id)
else:
if not comments_node:
raise exceptions.DataError(
"if comments_node is present, it must not be empty"
)
for comments_data in mb_data['comments']:
uri = comments_data.get('uri')
comments_node = comments_data.get('node')
try:
comments_service = jid.JID(comments_data["service"])
except KeyError:
comments_service = None
try:
comments_service = jid.JID(mb_data["comments_service"])
except KeyError:
comments_service = yield self.getCommentsService(client, service)
try:
yield self._p.createNode(client, comments_service, comments_node, options)
except error.StanzaError as e:
if e.condition == "conflict":
log.info(
"node {} already exists on service {}".format(
comments_node, comments_service
)
)
if uri:
uri_service, uri_node = self.parseCommentUrl(uri)
if ((comments_node is not None and comments_node!=uri_node)
or (comments_service is not None and comments_service!=uri_service)):
raise ValueError(
f"Incoherence between comments URI ({uri}) and comments_service "
f"({comments_service}) or comments_node ({comments_node})")
comments_data['service'] = comments_service = uri_service
comments_data['node'] = comments_node = uri_node
else:
raise e
else:
if access == self._p.ACCESS_WHITELIST:
# for whitelist access we need to copy affiliations from parent item
comments_affiliations = yield self._p.getNodeAffiliations(
client, service, node
)
# …except for "member", that we transform to publisher
# because we wants members to be able to write to comments
for jid_, affiliation in list(comments_affiliations.items()):
if affiliation == "member":
comments_affiliations[jid_] == "publisher"
if not comments_node:
comments_node = self.getCommentsNode(item_id)
comments_data['node'] = comments_node
if comments_service is None:
comments_service = yield self.getCommentsService(client, service)
if comments_service is None:
comments_service = client.jid.userhostJID()
comments_data['service'] = comments_service
yield self._p.setNodeAffiliations(
client, comments_service, comments_node, comments_affiliations
comments_data['uri'] = xmpp_uri.buildXMPPUri(
"pubsub",
path=comments_service.full(),
node=comments_node,
)
if comments_service is None:
comments_service = client.jid.userhostJID()
try:
yield self._p.createNode(client, comments_service, comments_node, options)
except error.StanzaError as e:
if e.condition == "conflict":
log.info(
"node {} already exists on service {}".format(
comments_node, comments_service
)
)
else:
raise e
else:
if access == self._p.ACCESS_WHITELIST:
# for whitelist access we need to copy affiliations from parent item
comments_affiliations = yield self._p.getNodeAffiliations(
client, service, node
)
# …except for "member", that we transform to publisher
# because we wants members to be able to write to comments
for jid_, affiliation in list(comments_affiliations.items()):
if affiliation == "member":
comments_affiliations[jid_] == "publisher"
if "comments" in mb_data:
if not mb_data["comments"]:
raise exceptions.DataError(
"if comments is present, it must not be empty"
)
if "comments_node" in mb_data or "comments_service" in mb_data:
raise exceptions.DataError(
"You can't use comments_service/comments_node and comments at the "
"same time"
)
else:
mb_data["comments"] = self._p.getNodeURI(comments_service, comments_node)
yield self._p.setNodeAffiliations(
client, comments_service, comments_node, comments_affiliations
)
def _mbSend(self, service, node, data, profile_key):
service = jid.JID(service) if service else None
@ -796,8 +812,8 @@ class XEP_0277(object):
def _mbGetSerialise(self, data):
items, metadata = data
items = [data_format.serialise(item) for item in items]
return items, metadata
metadata['items'] = items
return data_format.serialise(metadata)
def _mbGet(self, service="", node="", max_items=10, item_ids=None, extra_dict=None,
profile_key=C.PROF_KEY_NONE):
@ -840,7 +856,8 @@ class XEP_0277(object):
rsm_request=rsm_request,
extra=extra,
)
mb_data = yield self._p.transItemsDataD(items_data, self.item2mbdata)
mb_data = yield self._p.transItemsDataD(
items_data, self.item2mbdata)
defer.returnValue(mb_data)
def parseCommentUrl(self, node_url):
@ -1259,7 +1276,8 @@ class XEP_0277(object):
client, service, node, max_items, rsm_request=rsm_request, extra=extra
)
d.addCallback(
lambda items_data: self._p.transItemsDataD(items_data, self.item2mbdata)
lambda items_data: self._p.transItemsDataD(
items_data, self.item2mbdata)
)
d.addCallback(getComments)
d.addCallback(lambda items_comments_data: ("", items_comments_data))

28
sat/plugins/plugin_xep_0313.py
View file

@ -216,24 +216,6 @@ class XEP_0313(object):
return mam.MAMRequest(**mam_args) if mam_args else None
def serialise(self, mam_response, data=None):
"""Serialise data for MAM
Key set in data can be:
- mam_complete: a bool const indicating if all items have been received
- mam_stable: a bool const which is False if items order may be changed
All values are set as strings.
@param mam_response(dict): response data to serialise
@param data(dict, None): dict to update with mam_* data.
If None, a new dict is created
@return (dict): data dict
"""
if data is None:
data = {}
data["mam_complete"] = C.boolConst(mam_response['complete'])
data["mam_stable"] = C.boolConst(mam_response['stable'])
return data
def getMessageFromResult(self, client, mess_elt, mam_req, service=None):
"""Extract usable <message/> from MAM query result
@ -345,8 +327,10 @@ class XEP_0313(object):
service=service)
mess_data = client.messageProt.parseMessage(fwd_message_elt)
mess_list.append(client.messageGetBridgeArgs(mess_data))
metadata = self._rsm.serialise(rsm_response)
self.serialise(mam_response, metadata)
metadata = {
'rsm': self._rsm.response2dict(rsm_response),
'mam': mam_response
}
return mess_list, metadata, client.profile
def _getArchives(self, service, extra_ser, profile_key):
@ -354,8 +338,8 @@ class XEP_0313(object):
@return: tuple with:
- list of message with same data as in bridge.messageNew
- response metadata with:
- rsm data (rsm_first, rsm_last, rsm_count, rsm_index)
- mam data (mam_complete, mam_stable)
- rsm data (first, last, count, index)
- mam data (complete, stable)
- profile
"""
client = self.host.getClient(profile_key)

42
sat_frontends/jp/cmd_blog.py
View file

@ -33,7 +33,6 @@ from sat.core.i18n import _
from sat_frontends.jp.constants import Const as C
from sat_frontends.jp import common
from sat.tools.common.ansi import ANSI as A
from sat.tools.common import data_objects
from sat.tools.common import uri
from sat.tools import config
from configparser import NoSectionError, NoOptionError
@ -262,7 +261,9 @@ class Get(base.CommandBase):
# TODO: add MAM filters
def template_data_mapping(self, data):
return {"items": data_objects.BlogItems(data, deserialise=False)}
items, blog_items = data
blog_items['items'] = items
return {"blog_items": blog_items}
def format_comments(self, item, keys):
comments_data = data_format.dict2iterdict(
@ -424,22 +425,22 @@ class Get(base.CommandBase):
async def start(self):
try:
mb_result = await self.host.bridge.mbGet(
self.args.service,
self.args.node,
self.args.max,
self.args.items,
self.getPubsubExtra(),
self.profile
mb_data = data_format.deserialise(
await self.host.bridge.mbGet(
self.args.service,
self.args.node,
self.args.max,
self.args.items,
self.getPubsubExtra(),
self.profile
)
)
except Exception as e:
self.disp(f"can't get blog items: {e}", error=True)
self.host.quit(C.EXIT_BRIDGE_ERRBACK)
else:
items, metadata = mb_result
items = [data_format.deserialise(i) for i in items]
mb_result = items, metadata
await self.output(mb_result)
items = mb_data.pop('items')
await self.output((items, mb_data))
self.host.quit(C.EXIT_OK)
@ -592,14 +593,17 @@ class Edit(base.CommandBase, BlogPublishCommon, common.BaseEdit):
async def getItemData(self, service, node, item):
items = [item] if item else []
mb_data = await self.host.bridge.mbGet(
service, node, 1, items, {}, self.profile)
mb_data = data_format.deserialise(mb_data[0][0])
mb_data = data_format.deserialise(
await self.host.bridge.mbGet(
service, node, 1, items, {}, self.profile
)
)
item = mb_data['items'][0]
try:
content = mb_data["content_xhtml"]
content = item["content_xhtml"]
except KeyError:
content = mb_data["content"]
content = item["content"]
if content:
content = await self.host.bridge.syntaxConvert(
content, "text", SYNTAX_XHTML, False, self.profile
@ -623,7 +627,7 @@ class Edit(base.CommandBase, BlogPublishCommon, common.BaseEdit):
root = etree.fromstring(content, parser)
content = etree.tostring(root, encoding=str, pretty_print=True)
return content, mb_data, mb_data["id"]
return content, item, item["id"]
async def start(self):
# if there are user defined extension, we use them