erebion/libervia-backend
Watch
1
0
Fork
You've already forked libervia-backend
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

backend, frontend: messages refactoring (huge commit, not finished):

Browse source 
/!\ database schema has been modified, do a backup before updating

    message have been refactored, here are the main changes:

    - languages are now handled
    - all messages have an uid (internal to SàT)
    - message updating is anticipated
    - subject is now first class
    - new naming scheme is used newMessage => messageNew, getHistory => historyGet, sendMessage => messageSend
    - minimal compatibility refactoring in quick_frontend/Primitivus, better refactoring should follow
    - threads handling
    - delayed messages are saved into history
    - info messages may also be saved in history (e.g. to keep track of people joining/leaving a room)
    - duplicate messages should be avoided
    - historyGet return messages in right order, no need to sort again
    - plugins have been updated to follow new features, some of them need to be reworked (e.g. OTR)
    - XEP-0203 (Delayed Delivery) is now fully handled in core, the plugin just handle disco and creation of a delay element
    - /!\ jp and Libervia are currently broken, as some features of Primitivus

    It has been put in one huge commit to avoid breaking messaging between changes.
    This is the main part of message refactoring, other commits will follow to take profit of the new features/behaviour.
This commit is contained in:
Goffi 2016-05-24 22:11:04 +02:00
commit f12803fa65
2 changed files with 784 additions and 453 deletions
Download patch file Download diff file Expand all files Collapse all files

187
src/core/sat_main.py
View file

@ -40,6 +40,7 @@ from uuid import uuid4
import sys
import os.path
import uuid
import time
try:
from collections import OrderedDict # only available from python 2.7
@ -85,7 +86,7 @@ class SAT(service.Service):
self.bridge.register("getPresenceStatuses", self.memory._getPresenceStatuses)
self.bridge.register("getWaitingSub", self.memory.getWaitingSub)
self.bridge.register("getWaitingConf", self.getWaitingConf)
self.bridge.register("sendMessage", self._sendMessage)
self.bridge.register("messageSend", self._messageSend)
self.bridge.register("getConfig", self._getConfig)
self.bridge.register("setParam", self.setParam)
self.bridge.register("getParamA", self.memory.getStringParamA)
@ -94,7 +95,7 @@ class SAT(service.Service):
self.bridge.register("getParamsUI", self.memory.getParamsUI)
self.bridge.register("getParamsCategories", self.memory.getParamsCategories)
self.bridge.register("paramsRegisterApp", self.memory.paramsRegisterApp)
self.bridge.register("getHistory", self.memory.getHistory)
self.bridge.register("historyGet", self.memory.historyGet)
self.bridge.register("setPresence", self._setPresence)
self.bridge.register("subscription", self.subscription)
self.bridge.register("addContact", self._addContact)
@ -527,74 +528,123 @@ class SAT(service.Service):
ret.append((conf_id, conf_type, data))
return ret
def generateMessageXML(self, mess_data):
mess_data['xml'] = domish.Element((None, 'message'))
mess_data['xml']["to"] = mess_data["to"].full()
mess_data['xml']["from"] = mess_data['from'].full()
mess_data['xml']["type"] = mess_data["type"]
mess_data['xml']['id'] = str(uuid4())
if mess_data["subject"]:
mess_data['xml'].addElement("subject", None, mess_data['subject'])
if mess_data["message"]: # message without body are used to send chat states
mess_data['xml'].addElement("body", None, mess_data["message"])
return mess_data
def generateMessageXML(self, data):
"""Generate <message/> stanza from message data
def _sendMessage(self, to_s, msg, subject=None, mess_type='auto', extra={}, profile_key=C.PROF_KEY_NONE):
to_jid = jid.JID(to_s)
@param data(dict): message data
domish element will be put in data['xml']
following keys are needed:
- from
- to
- uid: can be set to '' if uid attribute is not wanted
- message
- type
- subject
- extra
@return (dict) message data
"""
data['xml'] = message_elt = domish.Element((None, 'message'))
message_elt["to"] = data["to"].full()
message_elt["from"] = data['from'].full()
message_elt["type"] = data["type"]
if data['uid']: # key must be present but can be set to ''
# by a plugin to avoid id on purpose
message_elt['id'] = data['uid']
for lang, subject in data["subject"].iteritems():
subject_elt = message_elt.addElement("subject", content=subject)
if lang:
subject_elt['xml:lang'] = lang
for lang, message in data["message"].iteritems():
body_elt = message_elt.addElement("body", content=message)
if lang:
body_elt['xml:lang'] = lang
try:
thread = data['extra']['thread']
except KeyError:
if 'thread_parent' in data['extra']:
raise exceptions.InternalError(u"thread_parent found while there is not associated thread")
else:
thread_elt = message_elt.addElement("thread", content=thread)
try:
thread_elt["parent"] = data["extra"]["thread_parent"]
except KeyError:
pass
return data
def _messageSend(self, to_jid_s, message, subject=None, mess_type='auto', extra=None, profile_key=C.PROF_KEY_NONE):
client = self.getClient(profile_key)
to_jid = jid.JID(to_jid_s)
#XXX: we need to use the dictionary comprehension because D-Bus return its own types, and pickle can't manage them. TODO: Need to find a better way
return self.sendMessage(to_jid, msg, subject, mess_type, {unicode(key): unicode(value) for key, value in extra.items()}, profile_key=profile_key)
return self.messageSend(client, to_jid, message, subject, mess_type, {unicode(key): unicode(value) for key, value in extra.items()})
def sendMessage(self, to_jid, msg, subject=None, mess_type='auto', extra={}, no_trigger=False, profile_key=C.PROF_KEY_NONE):
#FIXME: check validity of recipient
profile = self.memory.getProfileName(profile_key)
assert profile
client = self.profiles[profile]
def messageSend(self, client, to_jid, message, subject=None, mess_type='auto', extra=None, uid=None, no_trigger=False):
"""Send a message to an entity
@param to_jid(jid.JID): destinee of the message
@param message(dict): message body, key is the language (use '' when unknown)
@param subject(dict): message subject, key is the language (use '' when unknown)
@param mess_type(str): one of standard message type (cf RFC 6121 §5.2.2) or:
- auto: for automatic type detection
- info: for information ("info_type" can be specified in extra)
@param extra(dict, None): extra data. Key can be:
- info_type: information type, can be
TODO
@param uid(unicode, None): unique id:
should be unique at least in this XMPP session
if None, an uuid will be generated
@param no_trigger (bool): if True, messageSend trigger will no be used
useful when a message need to be sent without any modification
"""
profile = client.profile
if subject is None:
subject = {}
if extra is None:
extra = {}
mess_data = { # we put data in a dict, so trigger methods can change them
"to": to_jid,
data = { # dict is similar to the one used in client.onMessage
"from": client.jid,
"message": msg,
"to": to_jid,
"uid": uid or unicode(uuid.uuid4()),
"message": message,
"subject": subject,
"type": mess_type,
"extra": extra,
"timestamp": time.time(),
}
pre_xml_treatments = defer.Deferred() # XXX: plugin can add their pre XML treatments to this deferred
post_xml_treatments = defer.Deferred() # XXX: plugin can add their post XML treatments to this deferred
if mess_data["type"] == "auto":
if data["type"] == "auto":
# we try to guess the type
if mess_data["subject"]:
mess_data["type"] = 'normal'
elif not mess_data["to"].resource: # if to JID has a resource, the type is not 'groupchat'
if data["subject"]:
data["type"] = 'normal'
elif not data["to"].resource: # if to JID has a resource, the type is not 'groupchat'
# we may have a groupchat message, we check if the we know this jid
try:
entity_type = self.memory.getEntityData(mess_data["to"], ['type'], profile)["type"]
entity_type = self.memory.getEntityData(data["to"], ['type'], profile)["type"]
#FIXME: should entity_type manage resources ?
except (exceptions.UnknownEntityError, KeyError):
entity_type = "contact"
if entity_type == "chatroom":
mess_data["type"] = 'groupchat'
data["type"] = 'groupchat'
else:
mess_data["type"] = 'chat'
data["type"] = 'chat'
else:
mess_data["type"] == 'chat'
mess_data["type"] == "chat" if mess_data["subject"] else "normal"
data["type"] == 'chat'
data["type"] == "chat" if data["subject"] else "normal"
send_only = mess_data['extra'].get('send_only', None)
# FIXME: send_only is used by libervia's OTR plugin to avoid
# the triggers from frontend, and no_trigger do the same
# thing internally, this could be unified
send_only = data['extra'].get('send_only', None)
if not no_trigger and not send_only:
if not self.trigger.point("sendMessage", mess_data, pre_xml_treatments, post_xml_treatments, profile):
if not self.trigger.point("messageSend", client, data, pre_xml_treatments, post_xml_treatments):
return defer.succeed(None)
log.debug(_(u"Sending message (type {type}, to {to})").format(type=mess_data["type"], to=to_jid.full()))
log.debug(_(u"Sending message (type {type}, to {to})").format(type=data["type"], to=to_jid.full()))
def cancelErrorTrap(failure):
"""A message sending can be cancelled by a plugin treatment"""
failure.trap(exceptions.CancelError)
pre_xml_treatments.addCallback(lambda dummy: self.generateMessageXML(mess_data))
pre_xml_treatments.addCallback(lambda dummy: self.generateMessageXML(data))
pre_xml_treatments.chainDeferred(post_xml_treatments)
post_xml_treatments.addCallback(self._sendMessageToStream, client)
if send_only:
@ -602,58 +652,53 @@ class SAT(service.Service):
else:
post_xml_treatments.addCallback(self._storeMessage, client)
post_xml_treatments.addCallback(self.sendMessageToBridge, client)
post_xml_treatments.addErrback(cancelErrorTrap)
pre_xml_treatments.callback(mess_data)
post_xml_treatments.addErrback(self._cancelErrorTrap)
pre_xml_treatments.callback(data)
return pre_xml_treatments
def _sendMessageToStream(self, mess_data, client):
def _cancelErrorTrap(failure):
"""A message sending can be cancelled by a plugin treatment"""
failure.trap(exceptions.CancelError)
def _sendMessageToStream(self, data, client):
"""Actualy send the message to the server
@param mess_data: message data dictionnary
@param data: message data dictionnary
@param client: profile's client
"""
client.xmlstream.send(mess_data['xml'])
return mess_data
client.xmlstream.send(data['xml'])
return data
def _storeMessage(self, mess_data, client):
def _storeMessage(self, data, client):
"""Store message into database (for local history)
@param mess_data: message data dictionnary
@param data: message data dictionnary
@param client: profile's client
"""
if mess_data["type"] != "groupchat":
if data["type"] != C.MESS_TYPE_GROUPCHAT:
# we don't add groupchat message to history, as we get them back
# and they will be added then
if mess_data['message']: # we need a message to save something
self.memory.addToHistory(client.jid, mess_data['to'],
unicode(mess_data["message"]),
unicode(mess_data["type"]),
mess_data['extra'],
profile=client.profile)
if data['message']: # we need a message to save something
self.memory.addToHistory(client, data)
else:
log.warning(_("No message found")) # empty body should be managed by plugins before this point
return mess_data
log.warning(u"No message found") # empty body should be managed by plugins before this point
return data
def sendMessageToBridge(self, mess_data, client):
def sendMessageToBridge(self, data, client):
"""Send message to bridge, so frontends can display it
@param mess_data: message data dictionnary
@param data: message data dictionnary
@param client: profile's client
"""
if mess_data["type"] != "groupchat":
# we don't send groupchat message back to bridge, as we get them back
if data["type"] != C.MESS_TYPE_GROUPCHAT:
# we don't send groupchat message to bridge, as we get them back
# and they will be added the
if mess_data['message']: # we need a message to save something
# We send back the message, so all clients are aware of it
self.bridge.newMessage(mess_data['from'].full(),
unicode(mess_data["message"]),
mess_type=mess_data["type"],
to_jid=mess_data['to'].full(),
extra=mess_data['extra'],
profile=client.profile)
if data['message']: # we need a message to send something
# We send back the message, so all frontends are aware of it
self.bridge.messageNew(data['uid'], data['timestamp'], data['from'].full(), data['to'].full(), data['message'], data['subject'], data['type'], data['extra'], profile=client.profile)
else:
log.warning(_("No message found"))
return mess_data
return data
def _setPresence(self, to="", show="", statuses=None, profile_key=C.PROF_KEY_NONE):
return self.setPresence(jid.JID(to) if to else None, show, statuses, profile_key)

111
src/core/xmpp.py
View file

@ -20,14 +20,20 @@
from sat.core.i18n import _
from sat.core.constants import Const as C
from twisted.internet import task, defer
from twisted.words.protocols.jabber import jid, xmlstream
from twisted.words.protocols.jabber.xmlstream import XMPPHandler
from twisted.words.protocols.jabber import xmlstream
from twisted.words.protocols.jabber import error
from wokkel import client, disco, xmppim, generic, delay, iwokkel
from twisted.words.protocols.jabber import jid
from twisted.python import failure
from wokkel import client, disco, xmppim, generic, iwokkel
from wokkel import delay
from sat.core.log import getLogger
log = getLogger(__name__)
from sat.core import exceptions
from zope.interface import implements
from twisted.words.protocols.jabber.xmlstream import XMPPHandler
import time
import calendar
import uuid
class SatXMPPClient(client.XMPPClient):
@ -42,6 +48,7 @@ class SatXMPPClient(client.XMPPClient):
self.__connected = False
self.profile = profile
self.host_app = host_app
self._mess_id_uid = {} # map from message id to uid use in history. Key: (full_jid,message_id) Value: uid
self.conn_deferred = defer.Deferred()
self._waiting_conf = {} # callback called when a confirmation is received
self._progress_cb = {} # callback called when a progress is requested (key = progress id)
@ -131,53 +138,85 @@ class SatMessageProtocol(xmppim.MessageProtocol):
xmppim.MessageProtocol.__init__(self)
self.host = host
def onMessage(self, message):
if not message.hasAttribute('from'):
message['from'] = self.parent.jid.host
log.debug(_(u"got message from: %s") % message["from"])
def onMessage(self, message_elt):
# TODO: handle threads
client = self.parent
if not 'from' in message_elt.attributes:
message_elt['from'] = client.jid.host
log.debug(_(u"got message from: {from_}").format(from_=message_elt['from']))
post_treat = defer.Deferred() # XXX: plugin can add their treatments to this deferred
if not self.host.trigger.point("MessageReceived", message, post_treat, profile=self.parent.profile):
if not self.host.trigger.point("MessageReceived", client, message_elt, post_treat):
return
data = {"from": message['from'],
"to": message['to'],
"body": "",
"extra": {}}
message = {}
subject = {}
extra = {}
data = {"from": message_elt['from'],
"to": message_elt['to'],
"uid": message_elt.getAttribute('uid', unicode(uuid.uuid4())), # XXX: uid is not a standard attribute but may be added by plugins
"message": message,
"subject": subject,
"type": message_elt.getAttribute('type', 'normal'),
"extra": extra}
for e in message.elements():
if e.name == "body":
data['body'] = e.children[0] if e.children else ""
elif e.name == "subject" and e.children:
data['extra']['subject'] = e.children[0]
try:
data['stanza_id'] = message_elt['id']
except KeyError:
pass
else:
client._mess_id_uid[(data['from'], data['stanza_id'])] = data['uid']
data['type'] = message['type'] if message.hasAttribute('type') else 'normal'
# message
for e in message_elt.elements(C.NS_CLIENT, 'body'):
message[e.getAttribute('xml:lang','')] = unicode(e)
# subject
for e in message_elt.elements(C.NS_CLIENT, 'subject'):
subject[e.getAttribute('xml:lang','')] = unicode(e)
# delay and timestamp
try:
delay_elt = message_elt.elements(delay.NS_DELAY, 'delay').next()
except StopIteration:
data['timestamp'] = time.time()
else:
parsed_delay = delay.Delay.fromElement(delay_elt)
data['timestamp'] = calendar.timegm(parsed_delay.stamp.utctimetuple())
data['received_timestamp'] = unicode(time.time())
if parsed_delay.sender:
data['delay_sender'] = parsed_delay.sender.full()
def skipEmptyMessage(data):
if not data['message'] and not data['extra']:
raise failure.Failure(exceptions.CancelError())
return data
def bridgeSignal(data):
try:
data['extra']['received_timestamp'] = data['received_timestamp']
data['extra']['delay_sender'] = data['delay_sender']
except KeyError:
pass
if data is not None:
self.host.bridge.newMessage(data['from'], data['body'], data['type'], data['to'], data['extra'], profile=self.parent.profile)
self.host.bridge.messageNew(data['uid'], data['timestamp'], data['from'].full(), data['to'].full(), data['message'], data['subject'], data['type'], data['extra'], profile=client.profile)
return data
def addToHistory(data):
try:
timestamp = data['extra']['timestamp'] # timestamp added by XEP-0203
except KeyError:
self.host.memory.addToHistory(jid.JID(data['from']), jid.JID(data['to']), data['body'], data['type'], data['extra'], profile=self.parent.profile)
else:
if data['type'] != 'groupchat': # XXX: we don't save delayed messages in history for groupchats
#TODO: add delayed messages to history if they aren't already in it
data['extra']['archive'] = timestamp # FIXME: this "archive" is actually never used
self.host.memory.addToHistory(jid.JID(data['from']), jid.JID(data['to']), data['body'], data['type'], data['extra'], timestamp, profile=self.parent.profile)
data['from'] = jid.JID(data['from'])
data['to'] = jid.JID(data['to'])
self.host.memory.addToHistory(client, data)
return data
def treatmentsEb(failure):
failure.trap(exceptions.SkipHistory)
def treatmentsEb(failure_):
failure_.trap(exceptions.SkipHistory)
return data
def cancelErrorTrap(failure):
def cancelErrorTrap(failure_):
"""A message sending can be cancelled by a plugin treatment"""
failure.trap(exceptions.CancelError)
failure_.trap(exceptions.CancelError)
post_treat.addCallback(skipEmptyMessage)
post_treat.addCallback(addToHistory)
post_treat.addErrback(treatmentsEb)
post_treat.addCallback(bridgeSignal)
@ -506,7 +545,7 @@ class RegisteringAuthenticator(xmlstream.ConnectAuthenticator):
def connectionMade(self):
log.debug(_(u"Connection made with %s" % self.jabber_host))
self.xmlstream.namespace = "jabber:client"
self.xmlstream.namespace = C.NS_CLIENT
self.xmlstream.sendHeader()
iq = xmlstream.IQ(self.xmlstream, 'set')
@ -526,10 +565,10 @@ class RegisteringAuthenticator(xmlstream.ConnectAuthenticator):
log.debug(_(u"Registration answer: %s") % answer.toXml())
self.xmlstream.sendFooter()
def registrationFailure(self, failure):
log.info(_("Registration failure: %s") % unicode(failure.value))
def registrationFailure(self, failure_):
log.info(_("Registration failure: %s") % unicode(failure_.value))
self.xmlstream.sendFooter()
raise failure.value
raise failure_.value
class SatVersionHandler(generic.VersionHandler):