From 9647e909438f252fccb38a2186e774fc076f176b Mon Sep 17 00:00:00 2001 From: Goffi Date: Mon, 18 Jul 2016 00:52:02 +0200 Subject: [PATCH] backend (memory), frontends: improved history filtering: a "filters" dictionnary is now use to filter, it can have, for now, filtering on: - "body": filter only on the body (equivalent to former "search" parameter, but not case sensitive) - "search": fitler on body + source resource - "types": allowed types - "not_types": forbidden types primitivus now do searching using "search", i.e. source resource is now taken into account (and search is now case insensitive) --- frontends/src/bridge/DBus.py | 4 +- frontends/src/primitivus/chat.py | 10 ++--- frontends/src/primitivus/primitivus | 2 +- frontends/src/quick_frontend/quick_chat.py | 12 +++--- src/bridge/DBus.py | 6 +-- .../bridge_constructor/bridge_template.ini | 8 +++- src/memory/memory.py | 10 ++--- src/memory/sqlite.py | 42 +++++++++++++------ 8 files changed, 58 insertions(+), 36 deletions(-) diff --git a/frontends/src/bridge/DBus.py b/frontends/src/bridge/DBus.py index 6a2b703f..e743b6a9 100644 --- a/frontends/src/bridge/DBus.py +++ b/frontends/src/bridge/DBus.py @@ -508,14 +508,14 @@ class DBusBridgeFrontend(BridgeFrontend): kwargs['error_handler'] = error_handler return self.db_core_iface.getWaitingSub(profile_key, **kwargs) - def historyGet(self, from_jid, to_jid, limit, between=True, search='', profile="@NONE@", callback=None, errback=None): + def historyGet(self, from_jid, to_jid, limit, between=True, filters='', profile="@NONE@", callback=None, errback=None): if callback is None: error_handler = None else: if errback is None: errback = log.error error_handler = lambda err:errback(dbus_to_bridge_exception(err)) - return self.db_core_iface.historyGet(from_jid, to_jid, limit, between, search, profile, timeout=const_TIMEOUT, reply_handler=callback, error_handler=error_handler) + return self.db_core_iface.historyGet(from_jid, to_jid, limit, between, filters, profile, timeout=const_TIMEOUT, reply_handler=callback, error_handler=error_handler) def isConnected(self, profile_key="@DEFAULT@", callback=None, errback=None): if callback is None: diff --git a/frontends/src/primitivus/chat.py b/frontends/src/primitivus/chat.py index 2325e582..2c8af188 100644 --- a/frontends/src/primitivus/chat.py +++ b/frontends/src/primitivus/chat.py @@ -540,12 +540,12 @@ class Chat(PrimitivusWidget, quick_chat.QuickChat): except AttributeError: pass - def updateHistory(self, size=C.HISTORY_LIMIT_DEFAULT, search='', profile='@NONE@'): + def updateHistory(self, size=C.HISTORY_LIMIT_DEFAULT, filters=None, profile='@NONE@'): del self.mess_walker[:] - if search: - self.mess_walker.append(urwid.Text(_(u"Results for searching the globbing pattern: {}").format(search))) - self.mess_walker.append(urwid.Text(_(u"Type ':history ' to reset the chat history").format(search))) - super(Chat, self).updateHistory(size, search, profile) + if filters and 'search' in filters: + self.mess_walker.append(urwid.Text(_(u"Results for searching the globbing pattern: {}").format(filters['search']))) + self.mess_walker.append(urwid.Text(_(u"Type ':history ' to reset the chat history"))) + super(Chat, self).updateHistory(size, filters, profile) def _onHistoryPrinted(self): """Refresh or scroll down the focus after the history is printed""" diff --git a/frontends/src/primitivus/primitivus b/frontends/src/primitivus/primitivus index 0d364d7e..a04d43a6 100755 --- a/frontends/src/primitivus/primitivus +++ b/frontends/src/primitivus/primitivus @@ -130,7 +130,7 @@ class EditBar(sat_widgets.ModalEdit): if not pattern: self.host.notif_bar.addMessage(D_("Please specify the globbing pattern to search for")) else: - widget.updateHistory(size=C.HISTORY_LIMIT_NONE, search=pattern, profile=widget.profile) + widget.updateHistory(size=C.HISTORY_LIMIT_NONE, filters={'search': pattern}, profile=widget.profile) elif command == 'filter': # FIXME: filter is now only for current widget, # need to be able to set it globally or per widget diff --git a/frontends/src/quick_frontend/quick_chat.py b/frontends/src/quick_frontend/quick_chat.py index c7bfce0e..d151c27a 100644 --- a/frontends/src/quick_frontend/quick_chat.py +++ b/frontends/src/quick_frontend/quick_chat.py @@ -278,18 +278,18 @@ class QuickChat(quick_widgets.QuickWidget): return True return False - def updateHistory(self, size=C.HISTORY_LIMIT_DEFAULT, search='', profile='@NONE@'): + def updateHistory(self, size=C.HISTORY_LIMIT_DEFAULT, filters=None, profile='@NONE@'): """Called when history need to be recreated Remove all message from history then call historyPrint Must probably be overriden by frontend to clear widget @param size (int): number of messages - @param search (str): pattern to filter the history results + @param filters (str): patterns to filter the history results @param profile (str): %(doc_profile)s """ self._locked = True self.messages.clear() - self.historyPrint(size, search, profile) + self.historyPrint(size, filters, profile) def _onHistoryPrinted(self): """Method called when history is printed (or failed) @@ -301,13 +301,15 @@ class QuickChat(quick_widgets.QuickWidget): for data in self._cache: self.messageNew(*data) - def historyPrint(self, size=C.HISTORY_LIMIT_DEFAULT, search='', profile='@NONE@'): + def historyPrint(self, size=C.HISTORY_LIMIT_DEFAULT, filters=None, profile='@NONE@'): """Print the current history @param size (int): number of messages @param search (str): pattern to filter the history results @param profile (str): %(doc_profile)s """ + if filters is None: + filters = {} if size == 0: log.debug(u"Empty history requested, skipping") self._onHistoryPrinted() @@ -340,7 +342,7 @@ class QuickChat(quick_widgets.QuickWidget): log.error(_(u"Can't get history")) self._onHistoryPrinted() - self.host.bridge.historyGet(unicode(self.host.profiles[profile].whoami.bare), unicode(target), size, True, search, profile, callback=_historyGetCb, errback=_historyGetEb) + self.host.bridge.historyGet(unicode(self.host.profiles[profile].whoami.bare), unicode(target), size, True, filters, profile, callback=_historyGetCb, errback=_historyGetEb) def messageNew(self, uid, timestamp, from_jid, to_jid, msg, subject, type_, extra, profile): log.debug(u"messageNew ==> {}".format((uid, timestamp, from_jid, to_jid, msg, subject, type_, extra, profile))) diff --git a/src/bridge/DBus.py b/src/bridge/DBus.py index 41688456..65f2fb11 100755 --- a/src/bridge/DBus.py +++ b/src/bridge/DBus.py @@ -399,10 +399,10 @@ class DbusObject(dbus.service.Object): return self._callback("getWaitingSub", unicode(profile_key)) @dbus.service.method(const_INT_PREFIX+const_CORE_SUFFIX, - in_signature='ssibss', out_signature='a(sdssa{ss}a{ss}sa{ss})', + in_signature='ssiba{ss}s', out_signature='a(sdssa{ss}a{ss}sa{ss})', async_callbacks=('callback', 'errback')) - def historyGet(self, from_jid, to_jid, limit, between=True, search='', profile="@NONE@", callback=None, errback=None): - return self._callback("historyGet", unicode(from_jid), unicode(to_jid), limit, between, unicode(search), unicode(profile), callback=callback, errback=errback) + def historyGet(self, from_jid, to_jid, limit, between=True, filters='', profile="@NONE@", callback=None, errback=None): + return self._callback("historyGet", unicode(from_jid), unicode(to_jid), limit, between, filters, unicode(profile), callback=callback, errback=errback) @dbus.service.method(const_INT_PREFIX+const_CORE_SUFFIX, in_signature='s', out_signature='b', diff --git a/src/bridge/bridge_constructor/bridge_template.ini b/src/bridge/bridge_constructor/bridge_template.ini index 77c8a066..06159599 100644 --- a/src/bridge/bridge_constructor/bridge_template.ini +++ b/src/bridge/bridge_constructor/bridge_template.ini @@ -591,7 +591,7 @@ doc_param_2=app: name of the frontend registering the parameters async= type=method category=core -sig_in=ssibss +sig_in=ssiba{ss}s sig_out=a(sdssa{ss}a{ss}sa{ss}) param_3_default=True param_4_default='' @@ -601,7 +601,11 @@ doc_param_0=from_jid: source JID (bare jid for catch all, full jid else) doc_param_1=to_jid: dest JID (bare jid for catch all, full jid else) doc_param_2=limit: max number of history elements to get (0 for the whole history) doc_param_3=between: True if we want history between the two jids (in both direction), False if we only want messages from from_jid to to_jid -doc_param_4=search: pattern to filter the history results +doc_param_4=filters: patterns to filter the history results, can be: + - body: pattern must be in message body + - search: pattern must be in message body or source resource + - types: type must one of those, values are separated by spaces + - not_types: type must not be one of those, values are separated by spaces doc_param_5=%(doc_profile)s doc_return=Ordered list (by timestamp) of data as in [messageNew] (without final profile) diff --git a/src/memory/memory.py b/src/memory/memory.py index 78374b0c..446d43a2 100644 --- a/src/memory/memory.py +++ b/src/memory/memory.py @@ -513,10 +513,10 @@ class Memory(object): def addToHistory(self, client, data): return self.storage.addToHistory(data, client.profile) - def _historyGet(self, from_jid_s, to_jid_s, limit=C.HISTORY_LIMIT_NONE, between=True, search=None, profile=C.PROF_KEY_NONE): - return self.historyGet(jid.JID(from_jid_s), jid.JID(to_jid_s), limit, between, search, profile) + def _historyGet(self, from_jid_s, to_jid_s, limit=C.HISTORY_LIMIT_NONE, between=True, filters=None, profile=C.PROF_KEY_NONE): + return self.historyGet(jid.JID(from_jid_s), jid.JID(to_jid_s), limit, between, filters, profile) - def historyGet(self, from_jid, to_jid, limit=C.HISTORY_LIMIT_NONE, between=True, search=None, profile=C.PROF_KEY_NONE): + def historyGet(self, from_jid, to_jid, limit=C.HISTORY_LIMIT_NONE, between=True, filters=None, profile=C.PROF_KEY_NONE): """Retrieve messages in history @param from_jid (JID): source JID (full, or bare for catchall) @@ -526,7 +526,7 @@ class Memory(object): - C.HISTORY_LIMIT_NONE or None for unlimited - C.HISTORY_LIMIT_DEFAULT to use the HISTORY_LIMIT parameter value @param between (bool): confound source and dest (ignore the direction) - @param search (str): pattern to filter the history results + @param filters (str): pattern to filter the history results (see bridge API for details) @param profile (str): %(doc_profile)s @return (D(list)): list of message data as in [messageNew] """ @@ -537,7 +537,7 @@ class Memory(object): limit = None if limit == 0: return defer.succeed([]) - return self.storage.historyGet(from_jid, to_jid, limit, between, search, profile) + return self.storage.historyGet(from_jid, to_jid, limit, between, filters, profile) ## Statuses ## diff --git a/src/memory/sqlite.py b/src/memory/sqlite.py index 9bcee7e7..fe1fff6d 100644 --- a/src/memory/sqlite.py +++ b/src/memory/sqlite.py @@ -392,7 +392,7 @@ class SqliteStorage(object): ret.append((m['uid'], m['timestamp'], m['from'], m['to'], m['message'], m['subject'], m['type'], m['extra'])) return ret - def historyGet(self, from_jid, to_jid, limit=None, between=True, search=None, profile=None): + def historyGet(self, from_jid, to_jid, limit=None, between=True, filters=None, profile=None): """Retrieve messages in history @param from_jid (JID): source JID (full, or bare for catchall) @@ -406,10 +406,12 @@ class SqliteStorage(object): @return: list of tuple as in [messageNew] """ assert profile + if filters is None: + filters = {} if limit == 0: return defer.succeed([]) - query_parts = ["SELECT uid, update_uid, source, dest, source_res, dest_res, timestamp, received_timestamp,\ + query_parts = [u"SELECT uid, update_uid, source, dest, source_res, dest_res, timestamp, received_timestamp,\ type, extra, message, message.language, subject, subject.language, thread_id, thread.parent_id\ FROM history LEFT JOIN message ON history.uid = message.history_uid\ LEFT JOIN subject ON history.uid=subject.history_uid\ @@ -421,29 +423,43 @@ class SqliteStorage(object): values.append(_jid.userhost()) if _jid.resource: values.append(_jid.resource) - return '(%s=? AND %s_res=?)' % (type_, type_) - return '%s=?' % (type_, ) + return u'(%s=? AND %s_res=?)' % (type_, type_) + return u'%s=?' % (type_, ) if between: - query_parts.append("((%s AND %s) OR (%s AND %s))" % (test_jid('source', from_jid), + query_parts.append(u"((%s AND %s) OR (%s AND %s))" % (test_jid('source', from_jid), test_jid('dest', to_jid), test_jid('source', to_jid), test_jid('dest', from_jid))) else: - query_parts.append("%s AND %s" % (test_jid('source', from_jid), + query_parts.append(u"%s AND %s" % (test_jid('source', from_jid), test_jid('dest', to_jid))) - if search: - # TODO: use REGEXP (function to be defined) instead of GLOB: https://www.sqlite.org/lang_expr.html - query_parts.append("AND message GLOB ?") - values.append("*%s*" % search) - query_parts.append("ORDER BY timestamp DESC") # we reverse the order in sqliteHistoryToList + if filters: + if 'body' in filters: + # TODO: use REGEXP (function to be defined) instead of GLOB: https://www.sqlite.org/lang_expr.html + query_parts.append(u"AND message LIKE ?") + values.append(u"%{}%".format(filters['body'])) + if 'search' in filters: + query_parts.append(u"AND (message LIKE ? OR source_res LIKE ?)") + values.extend([u"%{}%".format(filters['search'])] * 2) + if 'types' in filters: + types = filters['types'].split() + query_parts.append(u"AND type IN ({})".format(u','.join("?"*len(types)))) + values.extend(types) + if 'not_types' in filters: + types = filters['not_types'].split() + query_parts.append(u"AND type NOT IN ({})".format(u','.join("?"*len(types)))) + values.extend(types) + + + query_parts.append(u"ORDER BY timestamp DESC") # we reverse the order in sqliteHistoryToList # we use DESC here so LIMIT keep the last messages if limit is not None: - query_parts.append("LIMIT ?") + query_parts.append(u"LIMIT ?") values.append(limit) - d = self.dbpool.runQuery(" ".join(query_parts), values) + d = self.dbpool.runQuery(u" ".join(query_parts), values) d.addCallback(self.sqliteHistoryToList) d.addCallback(self.listDict2listTuple) return d