backend, frontend: messages refactoring (huge commit, not finished):

/!\ database schema has been modified, do a backup before updating

    message have been refactored, here are the main changes:

    - languages are now handled
    - all messages have an uid (internal to SàT)
    - message updating is anticipated
    - subject is now first class
    - new naming scheme is used newMessage => messageNew, getHistory => historyGet, sendMessage => messageSend
    - minimal compatibility refactoring in quick_frontend/Primitivus, better refactoring should follow
    - threads handling
    - delayed messages are saved into history
    - info messages may also be saved in history (e.g. to keep track of people joining/leaving a room)
    - duplicate messages should be avoided
    - historyGet return messages in right order, no need to sort again
    - plugins have been updated to follow new features, some of them need to be reworked (e.g. OTR)
    - XEP-0203 (Delayed Delivery) is now fully handled in core, the plugin just handle disco and creation of a delay element
    - /!\ jp and Libervia are currently broken, as some features of Primitivus

    It has been put in one huge commit to avoid breaking messaging between changes.
    This is the main part of message refactoring, other commits will follow to take profit of the new features/behaviour.

frontends/src/bridge/DBus.py

@@ -336,15 +336,6 @@ class DBusBridgeFrontend(BridgeFrontend):
             error_handler = lambda err:errback(dbus_to_bridge_exception(err))
         return self.db_core_iface.getFeatures(profile_key, timeout=const_TIMEOUT, reply_handler=callback, error_handler=error_handler)
 
-    def getHistory(self, from_jid, to_jid, limit, between=True, search='', profile="@NONE@", callback=None, errback=None):
-        if callback is None:
-            error_handler = None
-        else:
-            if errback is None:
-                errback = log.error
-            error_handler = lambda err:errback(dbus_to_bridge_exception(err))
-        return self.db_core_iface.getHistory(from_jid, to_jid, limit, between, search, profile, timeout=const_TIMEOUT, reply_handler=callback, error_handler=error_handler)
-
     def getMainResource(self, contact_jid, profile_key="@DEFAULT@", callback=None, errback=None):
         if callback is None:
             error_handler = None
@@ -517,6 +508,15 @@ class DBusBridgeFrontend(BridgeFrontend):
             kwargs['error_handler'] = error_handler
         return self.db_core_iface.getWaitingSub(profile_key, **kwargs)
 
+    def historyGet(self, from_jid, to_jid, limit, between=True, search='', profile="@NONE@", callback=None, errback=None):
+        if callback is None:
+            error_handler = None
+        else:
+            if errback is None:
+                errback = log.error
+            error_handler = lambda err:errback(dbus_to_bridge_exception(err))
+        return self.db_core_iface.historyGet(from_jid, to_jid, limit, between, search, profile, timeout=const_TIMEOUT, reply_handler=callback, error_handler=error_handler)
+
     def isConnected(self, profile_key="@DEFAULT@", callback=None, errback=None):
         if callback is None:
             error_handler = None
@@ -554,6 +554,15 @@ class DBusBridgeFrontend(BridgeFrontend):
             kwargs['error_handler'] = error_handler
         return self.db_core_iface.loadParamsTemplate(filename, **kwargs)
 
+    def messageSend(self, to_jid, message, subject='', mess_type="auto", extra={}, profile_key="@NONE@", callback=None, errback=None):
+        if callback is None:
+            error_handler = None
+        else:
+            if errback is None:
+                errback = log.error
+            error_handler = lambda err:errback(dbus_to_bridge_exception(err))
+        return self.db_core_iface.messageSend(to_jid, message, subject, mess_type, extra, profile_key, timeout=const_TIMEOUT, reply_handler=callback, error_handler=error_handler)
+
     def paramsRegisterApp(self, xml, security_limit=-1, app='', callback=None, errback=None):
         if callback is None:
             error_handler = None
@@ -661,15 +670,6 @@ class DBusBridgeFrontend(BridgeFrontend):
             kwargs['error_handler'] = error_handler
         return self.db_core_iface.saveParamsTemplate(filename, **kwargs)
 
-    def sendMessage(self, to_jid, message, subject='', mess_type="auto", extra={}, profile_key="@NONE@", callback=None, errback=None):
-        if callback is None:
-            error_handler = None
-        else:
-            if errback is None:
-                errback = log.error
-            error_handler = lambda err:errback(dbus_to_bridge_exception(err))
-        return self.db_core_iface.sendMessage(to_jid, message, subject, mess_type, extra, profile_key, timeout=const_TIMEOUT, reply_handler=callback, error_handler=error_handler)
-
     def setParam(self, name, value, category, security_limit=-1, profile_key="@DEFAULT@", callback=None, errback=None):
         if callback is None:
             error_handler = None

frontends/src/primitivus/chat.py

@@ -88,7 +88,6 @@ class Chat(PrimitivusWidget, QuickChat):
         self.text_list = urwid.ListBox(self.content)
         self.chat_widget = urwid.Frame(self.text_list)
         self.chat_colums = urwid.Columns([('weight', 8, self.chat_widget)])
-        self.chat_colums = urwid.Columns([('weight', 8, self.chat_widget)])
         self.pile = urwid.Pile([self.chat_colums])
         PrimitivusWidget.__init__(self, self.pile, self.target)
 

frontends/src/primitivus/primitivus

@@ -96,12 +96,13 @@ class EditBar(sat_widgets.ModalEdit):
         if self.mode == C.MODE_INSERTION:
             if isinstance(self.host.selected_widget, quick_chat.QuickChat):
                 chat_widget = self.host.selected_widget
-                self.host.sendMessage(chat_widget.target,
-                                     editBar.get_edit_text(),
-                                     mess_type = "groupchat" if chat_widget.type == 'group' else "chat", # TODO: put this in QuickChat
-                                     errback=lambda failure: self.host.notify(_("Error while sending message ({})").format(failure)),
-                                     profile_key=chat_widget.profile
-                                    )
+                self.host.messageSend(
+                    chat_widget.target,
+                    {'': editBar.get_edit_text()}, # TODO: handle language
+                    mess_type = "groupchat" if chat_widget.type == 'group' else "chat", # TODO: put this in QuickChat
+                    errback=lambda failure: self.host.notify(_("Error while sending message ({})").format(failure)),
+                    profile_key=chat_widget.profile
+                    )
                 editBar.set_edit_text('')
         elif self.mode == C.MODE_COMMAND:
             self.commandHandler()
@@ -723,7 +724,9 @@ class PrimitivusApp(QuickApp, InputHistory):
 
     def roomJoinedHandler(self, room_jid_s, room_nicks, user_nick, profile):
         super(PrimitivusApp, self).roomJoinedHandler(room_jid_s, room_nicks, user_nick, profile)
-        self.contact_lists[profile].setFocus(jid.JID(room_jid_s), True)
+        for contact_list in self.widgets.getWidgets(ContactList):
+            if profile in contact_list.profiles:
+                contact_list.setFocus(jid.JID(room_jid_s), True)
 
     def progressStartedHandler(self, pid, metadata, profile):
         super(PrimitivusApp, self).progressStartedHandler(pid, metadata, profile)

frontends/src/quick_frontend/quick_app.py

@@ -248,7 +248,7 @@ class QuickApp(object):
         self.registerSignal("disconnected")
         self.registerSignal("actionNew")
         self.registerSignal("newContact")
-        self.registerSignal("newMessage")
+        self.registerSignal("messageNew")
         self.registerSignal("newAlert")
         self.registerSignal("presenceUpdate")
         self.registerSignal("subscribe")
@@ -482,10 +482,10 @@ class QuickApp(object):
         groups = list(groups)
         self.contact_lists[profile].setContact(entity, groups, attributes, in_roster=True)
 
-    def newMessageHandler(self, from_jid_s, msg, type_, to_jid_s, extra, profile):
+    def messageNewHandler(self, uid, timestamp, from_jid_s, to_jid_s, msg, subject, type_, extra, profile):
         from_jid = jid.JID(from_jid_s)
         to_jid = jid.JID(to_jid_s)
-        if not self.trigger.point("newMessageTrigger", from_jid, msg, type_, to_jid, extra, profile=profile):
+        if not self.trigger.point("messageNewTrigger", uid, timestamp, from_jid, to_jid, msg, subject, type_, extra, profile=profile):
             return
 
         from_me = from_jid.bare == self.profiles[profile].whoami.bare
@@ -505,7 +505,8 @@ class QuickApp(object):
             contact_list.setContact(from_jid)
 
         # we display the message in the widget
-        chat_widget.newMessage(from_jid, target, msg, type_, extra, profile)
+
+        chat_widget.messageNew(uid, timestamp, from_jid, target, msg, subject, type_, extra, profile)
 
         # ContactList alert
         if not from_me:
@@ -520,16 +521,20 @@ class QuickApp(object):
             else:
                 contact_list.addAlert(from_jid.bare if type_ == C.MESS_TYPE_GROUPCHAT else from_jid)
 
-    def sendMessage(self, to_jid, message, subject='', mess_type="auto", extra={}, callback=None, errback=None, profile_key=C.PROF_KEY_NONE):
+    def messageSend(self, to_jid, message, subject=None, mess_type="auto", extra=None, callback=None, errback=None, profile_key=C.PROF_KEY_NONE):
+        if subject is None:
+            subject = {}
+        if extra is None:
+            extra = {}
         if callback is None:
             callback = lambda dummy=None: None # FIXME: optional argument is here because pyjamas doesn't support callback without arg with json proxy
         if errback is None:
             errback = lambda failure: self.showDialog(failure.fullname, failure.message, "error")
 
-        if not self.trigger.point("sendMessageTrigger", to_jid, message, subject, mess_type, extra, callback, errback, profile_key=profile_key):
+        if not self.trigger.point("messageSendTrigger", to_jid, message, subject, mess_type, extra, callback, errback, profile_key=profile_key):
             return
 
-        self.bridge.sendMessage(unicode(to_jid), message, subject, mess_type, extra, profile_key, callback=callback, errback=errback)
+        self.bridge.messageSend(unicode(to_jid), message, subject, mess_type, extra, profile_key, callback=callback, errback=errback)
 
     def newAlertHandler(self, msg, title, alert_type, profile):
         assert alert_type in ['INFO', 'ERROR']

frontends/src/quick_frontend/quick_chat.py

@@ -25,7 +25,6 @@ from sat_frontends.quick_frontend import quick_widgets
 from sat_frontends.quick_frontend.constants import Const as C
 from collections import OrderedDict
 from datetime import datetime
-from time import time
 
 try:
     # FIXME: to be removed when an acceptable solution is here
@@ -52,18 +51,15 @@ class QuickChat(quick_widgets.QuickWidget):
         self.nick = None
         self.games = {}  # key=game name (unicode), value=instance of quick_games.RoomGame
 
-        if type_ == C.CHAT_ONE2ONE:
-            self.historyPrint(profile=self.profile)
-
-        # FIXME: has been introduced to temporarily fix http://bugs.goffi.org/show_bug.cgi?id=12
-        self.initialised = False
+        self.historyPrint(profile=self.profile)
 
     def __str__(self):
         return u"Chat Widget [target: {}, type: {}, profile: {}]".format(self.target, self.type, self.profile)
 
     @staticmethod
-    def getWidgetHash(target, profile):
-        return (unicode(profile), target.bare)
+    def getWidgetHash(target, profiles):
+        profile = profiles[0]
+        return (profile, target.bare)
 
     @staticmethod
     def getPrivateHash(target, profile):
@@ -99,7 +95,7 @@ class QuickChat(quick_widgets.QuickWidget):
         """Tell if this chat widget manage an entity and message type couple
 
         @param entity (jid.JID): (full) jid of the sending entity
-        @param mess_type (str): message type as given by newMessage
+        @param mess_type (str): message type as given by messageNew
         @return (bool): True if this Chat Widget manage this couple
         """
         if self.type == C.CHAT_GROUP:
@@ -112,8 +108,6 @@ class QuickChat(quick_widgets.QuickWidget):
 
     def addUser(self, nick):
         """Add user if it is not in the group list"""
-        if not self.initialised:
-            return  # FIXME: tmp fix for bug 12, do not flood the room with the messages when we've just entered it
         self.printInfo("=> %s has joined the room" % nick)
 
     def removeUser(self, nick):
@@ -141,6 +135,7 @@ class QuickChat(quick_widgets.QuickWidget):
 
     def historyPrint(self, size=C.HISTORY_LIMIT_DEFAULT, search='', profile='@NONE@'):
         """Print the current history
+
         @param size (int): number of messages
         @param search (str): pattern to filter the history results
         @param profile (str): %(doc_profile)s
@@ -152,28 +147,26 @@ class QuickChat(quick_widgets.QuickWidget):
 
         target = self.target.bare
 
-        def onHistory(history):
-            self.initialised = True  # FIXME: tmp fix for bug 12
+        def _historyGetCb(history):
             day_format = "%A, %d %b %Y"  # to display the day change
             previous_day = datetime.now().strftime(day_format)
-            for line in history:
-                timestamp, from_jid, to_jid, message, type_, extra = line  # FIXME: extra is unused !
+            for data in history:
+                uid, timestamp, from_jid, to_jid, message, subject, type_, extra = data  # FIXME: extra is unused !
                 if ((self.type == C.CHAT_GROUP and type_ != C.MESS_TYPE_GROUPCHAT) or
                    (self.type == C.CHAT_ONE2ONE and type_ == C.MESS_TYPE_GROUPCHAT)):
                     continue
-                message_day = datetime.fromtimestamp(float(timestamp or time())).strftime(day_format)
+                message_day = datetime.fromtimestamp(timestamp).strftime(day_format)
                 if previous_day != message_day:
                     self.printDayChange(message_day)
                     previous_day = message_day
                 extra["timestamp"] = timestamp
-                self.newMessage(jid.JID(from_jid), target, message, type_, extra, profile)
+                self.messageNew(uid, timestamp, jid.JID(from_jid), target, message, subject, type_, extra, profile)
             self.afterHistoryPrint()
 
-        def onHistoryError(err):
+        def _historyGetEb(err):
             log.error(_("Can't get history"))
 
-        self.initialised = False  # FIXME: tmp fix for bug 12, here needed for :history and :search commands
-        self.host.bridge.getHistory(unicode(self.host.profiles[profile].whoami.bare), unicode(target), size, True, search, profile, callback=onHistory, errback=onHistoryError)
+        self.host.bridge.historyGet(unicode(self.host.profiles[profile].whoami.bare), unicode(target), size, True, search, profile, callback=_historyGetCb, errback=_historyGetEb)
 
     def _get_nick(self, entity):
         """Return nick of this entity when possible"""
@@ -195,34 +188,31 @@ class QuickChat(quick_widgets.QuickWidget):
         """
         return self.host.widgets.getOrCreateWidget(QuickChat, entity, type_=C.CHAT_ONE2ONE, force_hash=self.getPrivateHash(self.profile, entity), on_new_widget=self.onPrivateCreated, profile=self.profile) # we force hash to have a new widget, not this one again
 
-    def newMessage(self, from_jid, target, msg, type_, extra, profile):
+    def messageNew(self, uid, timestamp, from_jid, target, msg, subject, type_, extra, profile):
+        try:
+            msg = msg.itervalues().next() # FIXME: tmp fix until message refactoring is finished (msg is now a dict)
+        except StopIteration:
+            log.warning(u"No message found (uid: {})".format(uid))
+            msg = ''
         if self.type == C.CHAT_GROUP and target.resource and type_ != C.MESS_TYPE_GROUPCHAT:
             # we have a private message, we forward it to a private conversation widget
             chat_widget = self.getOrCreatePrivateWidget(target)
-            chat_widget.newMessage(from_jid, target, msg, type_, extra, profile)
+            chat_widget.messageNew(uid, timestamp, from_jid, target, msg, subject, type_, extra, profile)
             return
-        try:
-            timestamp = float(extra['timestamp'])
-        except KeyError:
-            timestamp = None
-
-        if not self.initialised and self.type == C.CHAT_ONE2ONE:
-            return  # FIXME: tmp fix for bug 12, do not display the first one2one message which is already in the local history
 
         if type_ == C.MESS_TYPE_INFO:
             self.printInfo(msg, extra=extra)
         else:
-            self.initialised = True  # FIXME: tmp fix for bug 12, do not discard any message from now
-
             nick = self._get_nick(from_jid)
             if msg.startswith('/me '):
-                self.printInfo('* %s %s' % (nick, msg[4:]), type_='me', extra=extra)
+                self.printInfo('* {} {}'.format(nick, msg[4:]), type_='me', extra=extra)
             else:
                 # my_message is True if message comes from local user
                 my_message = (from_jid.resource == self.nick) if self.type == C.CHAT_GROUP else (from_jid.bare == self.host.profiles[profile].whoami.bare)
                 self.printMessage(nick, my_message, msg, timestamp, extra, profile)
-        if timestamp:
-            self.afterHistoryPrint()
+        # FIXME: to be checked/removed after message refactoring
+        # if timestamp:
+        self.afterHistoryPrint()
 
     def printMessage(self, nick, my_message, message, timestamp, extra=None, profile=C.PROF_KEY_NONE):
         """Print message in chat window.
@@ -232,11 +222,12 @@ class QuickChat(quick_widgets.QuickWidget):
         @param message (unicode): message content
         @param extra (dict): extra data
         """
-        if not timestamp:
-            # XXX: do not send notifications for each line of the history being displayed
-            # FIXME: this must be changed in the future if the timestamp is passed with
-            # all messages and not only with the messages coming from the history.
-            self.notify(nick, message)
+        # FIXME: check/remove this if necessary (message refactoring)
+        # if not timestamp:
+        #     # XXX: do not send notifications for each line of the history being displayed
+        #     # FIXME: this must be changed in the future if the timestamp is passed with
+        #     # all messages and not only with the messages coming from the history.
+        self.notify(nick, message)
 
     def printInfo(self, msg, type_='normal', extra=None):
         """Print general info.

frontends/src/quick_frontend/quick_contact_list.py

@@ -549,6 +549,7 @@ class ProfileContactList(object):
             None to count all of them
         @return (list[unicode,None]): list of C.ALERT_* or None for undefined ones
         """
+        return [] # FIXME: temporarily disabled
         if not use_bare_jid:
             alerts = self._alerts.get(entity, [])
         else:

frontends/src/quick_frontend/quick_widgets.py

@@ -247,8 +247,8 @@ class QuickWidgetsManager(object):
 class QuickWidget(object):
     """generic widget base"""
     SINGLE=True # if True, there can be only one widget per target(s)
-    PROFILES_MULTIPLE=False
-    PROFILES_ALLOW_NONE=False
+    PROFILES_MULTIPLE=False # If True, this widget can handle several profiles at once
+    PROFILES_ALLOW_NONE=False # If True, this widget can be used without profile
 
     def __init__(self, host, target, profiles=None):
         """