jp: jp asyncio port:

/!\ this commit is huge. Jp is temporarily not working with `dbus` bridge /!\

This patch implements the port of jp to asyncio, so it is now correctly using the bridge
asynchronously, and it can be used with bridges like `pb`. This also simplify the code,
notably for things which were previously implemented with many callbacks (like pagination
with RSM).

During the process, some behaviours have been modified/fixed, in jp and backends, check
diff for details.
This commit is contained in:
Goffi 2019-09-25 08:56:41 +02:00
commit 76aadbb3fb
26 changed files with 3299 additions and 3300 deletions

View file

@ -334,6 +334,7 @@ class Const(object):
## Progress error special values ##
PROGRESS_ERROR_DECLINED = "declined" #  session has been declined by peer user
PROGRESS_ERROR_FAILED = "failed" #  something went wrong with the session
## Files ##
FILE_TYPE_DIRECTORY = "directory"

View file

@ -260,7 +260,7 @@ class SAT(service.Service):
)
self._unimport_plugin(plugin_path)
continue
except Exception as e:
except Exception:
import traceback
log.error(

View file

@ -86,11 +86,11 @@ class SatXMPPEntity(object):
self.actions = {} # used to keep track of actions for retrieval (key = action_id)
self.encryption = encryption.EncryptionHandler(self)
def __unicode__(self):
return "Client instance for profile {profile}".format(profile=self.profile)
def __str__(self):
return self.__unicode__.encode('utf-8')
return f"Client for profile {self.profile}"
def __repr__(self):
return f"{super().__repr__()} - profile: {self.profile!r}"
## initialisation ##

View file

@ -45,7 +45,7 @@ class BlockCipher(object):
@param key (unicode): the encryption key
@param text (unicode): the text to encrypt
@param leave_empty (bool): if True, empty text will be returned "as is"
@return: Deferred: base-64 encoded str
@return (D(str)): base-64 encoded encrypted message
"""
if leave_empty and text == "":
return succeed(text)
@ -59,6 +59,7 @@ class BlockCipher(object):
cipher = AES.new(key, AES.MODE_CFB, iv)
d = deferToThread(cipher.encrypt, BlockCipher.pad(text.encode("utf-8")))
d.addCallback(lambda ciphertext: b64encode(iv + ciphertext))
d.addCallback(lambda bytes_cypher: bytes_cypher.decode('utf-8'))
return d
@classmethod
@ -137,7 +138,7 @@ class PasswordHasher(object):
@return: Deferred: base-64 encoded str
"""
if leave_empty and password == "":
return succeed(b"")
return succeed("")
salt = (
b64decode(salt)[: PasswordHasher.SALT_LEN]
if salt
@ -145,11 +146,12 @@ class PasswordHasher(object):
)
d = deferToThread(PBKDF2, password, salt)
d.addCallback(lambda hashed: b64encode(salt + hashed))
d.addCallback(lambda hashed_bytes: hashed_bytes.decode('utf-8'))
return d
@classmethod
def compare_hash(cls, hashed_attempt, hashed):
assert isinstance(hashed, bytes)
assert isinstance(hashed, str)
return hashed_attempt == hashed
@classmethod
@ -160,7 +162,9 @@ class PasswordHasher(object):
@param hashed (str): the hash of the password
@return: Deferred: boolean
"""
assert isinstance(attempt, str)
assert isinstance(hashed, str)
leave_empty = hashed == ""
d = PasswordHasher.hash(attempt, hashed, leave_empty)
d.addCallback(cls.compare_hash, hashed=hashed.encode('utf-8'))
d.addCallback(cls.compare_hash, hashed=hashed)
return d

View file

@ -434,7 +434,7 @@ class Discovery(object):
extensions[form_type or ""] = fields
defer.returnValue((
disco_infos.features,
[str(f) for f in disco_infos.features],
[(cat, type_, name or "")
for (cat, type_), name in list(disco_infos.identities.items())],
extensions))

View file

@ -560,9 +560,11 @@ class Memory(object):
def initPersonalKey(__):
# be sure to call this after checking that the profile doesn't exist yet
# generated once for all and saved in a PersistentDict
personal_key = BlockCipher.getRandomKey(
base64=True
) # generated once for all and saved in a PersistentDict
).decode('utf-8')
self.auth_sessions.newSession(
{C.MEMORY_CRYPTO_KEY: personal_key}, profile=name
) # will be encrypted by setParam
@ -1260,8 +1262,8 @@ class Memory(object):
parent = current["parent"]
if not parent:
break
files_data = yield self.getFile(
self, client, peer_jid=None, file_id=parent, perms_to_check=None
files_data = yield self.getFiles(
client, peer_jid=None, file_id=parent, perms_to_check=None
)
try:
current = files_data[0]

View file

@ -529,17 +529,18 @@ class Params(object):
return d.addCallback(gotPlainPassword)
def __type_to_string(self, result):
""" convert result to string, according to its type """
def _type_to_str(self, result):
"""Convert result to string, according to its type """
if isinstance(result, bool):
return "true" if result else "false"
elif isinstance(result, int):
return C.boolConst(result)
elif isinstance(result, (list, set, tuple)):
return ', '.join(self._type_to_str(r) for r in result)
else:
return str(result)
return result
def getStringParamA(self, name, category, attr="value", profile_key=C.PROF_KEY_NONE):
""" Same as getParamA but for bridge: convert non string value to string """
return self.__type_to_string(
return self._type_to_str(
self.getParamA(name, category, attr, profile_key=profile_key)
)
@ -599,15 +600,10 @@ class Params(object):
return self._getAttr(node[1], attr, value)
def asyncGetStringParamA(
self,
name,
category,
attr="value",
security_limit=C.NO_SECURITY_LIMIT,
profile_key=C.PROF_KEY_NONE,
):
self, name, category, attr="value", security_limit=C.NO_SECURITY_LIMIT,
profile_key=C.PROF_KEY_NONE):
d = self.asyncGetParamA(name, category, attr, security_limit, profile_key)
d.addCallback(self.__type_to_string)
d.addCallback(self._type_to_str)
return d
def asyncGetParamA(
@ -957,14 +953,8 @@ class Params(object):
categories.append(cat.getAttribute("name"))
return categories
def setParam(
self,
name,
value,
category,
security_limit=C.NO_SECURITY_LIMIT,
profile_key=C.PROF_KEY_NONE,
):
def setParam(self, name, value, category, security_limit=C.NO_SECURITY_LIMIT,
profile_key=C.PROF_KEY_NONE):
"""Set a parameter, return None if the parameter is not in param xml.
Parameter of type 'password' that are not the SàT profile password are
@ -994,13 +984,11 @@ class Params(object):
return defer.succeed(None)
if not self.checkSecurityLimit(node[1], security_limit):
log.warning(
_(
"Trying to set parameter '%(param)s' in category '%(cat)s' without authorization!!!"
% {"param": name, "cat": category}
)
)
return defer.succeed(None)
msg = _(
f"{profile!r} is trying to set parameter {name!r} in category "
f"{category!r} without authorization!!!")
log.warning(msg)
raise exceptions.PermissionError(msg)
type_ = node[1].getAttribute("type")
if type_ == "int":
@ -1010,12 +998,10 @@ class Params(object):
try:
int(value)
except ValueError:
log.debug(
_(
"Trying to set parameter '%(param)s' in category '%(cat)s' with an non-integer value"
% {"param": name, "cat": category}
)
)
log.warning(_(
f"Trying to set parameter {name!r} in category {category!r} with"
f"an non-integer value"
))
return defer.succeed(None)
if node[1].hasAttribute("constraint"):
constraint = node[1].getAttribute("constraint")

View file

@ -236,7 +236,7 @@ class AdHocDBus(object):
flags=flags,
)
defer.returnValue((bus_name, methods))
defer.returnValue((str(bus_name), methods))
def _addCommand(self, client, adhoc_name, bus_name, methods, allowed_jids=None,
allowed_groups=None, allowed_magics=None, forbidden_jids=None,

View file

@ -381,7 +381,7 @@ class Events(object):
if register:
yield self.register(
client, service, None, {}, node, event_id, event_elt, creator=True)
client, None, {}, service, node, event_id, item_elt, creator=True)
defer.returnValue(node)
def _eventModify(self, service, node, id_, timestamp_update, data_update,
@ -447,24 +447,29 @@ class Events(object):
events.append((timestamp, data))
defer.returnValue(events)
def _eventInviteeGet(self, service, node, profile_key):
def _eventInviteeGet(self, service, node, invitee_jid_s, profile_key):
service = jid.JID(service) if service else None
node = node if node else NS_EVENT
client = self.host.getClient(profile_key)
return self.eventInviteeGet(client, service, node)
invitee_jid = jid.JID(invitee_jid_s) if invitee_jid_s else None
return self.eventInviteeGet(client, service, node, invitee_jid)
@defer.inlineCallbacks
def eventInviteeGet(self, client, service, node):
def eventInviteeGet(self, client, service, node, invitee_jid=None):
"""Retrieve attendance from event node
@param service(unicode, None): PubSub service
@param node(unicode): PubSub node of the event
@param node(unicode): PubSub node of the event's invitees
@param invitee_jid(jid.JId, None): jid of the invitee to retrieve (None to
retrieve profile's invitation). The bare jid correspond to the PubSub item id.
@return (dict): a dict with current attendance status,
an empty dict is returned if nothing has been answered yed
"""
if invitee_jid is None:
invitee_jid = client.jid
try:
items, metadata = yield self._p.getItems(
client, service, node, item_ids=[client.jid.userhost()]
client, service, node, item_ids=[invitee_jid.userhost()]
)
event_elt = next(items[0].elements(NS_EVENT, "invitee"))
except (exceptions.NotFound, IndexError):

View file

@ -18,11 +18,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from collections import Iterable
import copy
import itertools
from zope.interface import implementer
from twisted.words.protocols.jabber import jid
from twisted.words.protocols.jabber.xmlstream import XMPPHandler
from twisted.words.xish import domish
from twisted.internet import defer
from wokkel import disco, iwokkel
from wokkel import data_form
@ -174,7 +174,16 @@ class PubsubSchema(object):
)
elif isinstance(schema, data_form.Form):
if copy_form:
schema = copy.deepcopy(schema)
# XXX: we don't use deepcopy as it will do an infinite loop if a
# domish.Element is present in the form fields (happens for
# XEP-0315 data forms XML Element)
schema = data_form.Form(
formType = schema.formType,
title = schema.title,
instructions = schema.instructions[:],
formNamespace = schema.formNamespace,
fields = schema.fieldList,
)
defer.returnValue(schema)
try:
@ -394,6 +403,18 @@ class PubsubSchema(object):
if not values_list:
# if values don't map to allowed values, we use default ones
values_list = field.values
elif field.ext_type == 'xml':
# FIXME: XML elements are not handled correctly, we need to know if we
# actual XML/XHTML, or text to escape
for idx, value in enumerate(values_list[:]):
if not isinstance(value, domish.Element):
if field.value and field.value.uri == C.NS_XHTML:
div_elt = domish.Element((C.NS_XHTML, 'div'))
div_elt.addContent(str(value))
values_list[idx] = div_elt
else:
raise NotImplementedError
field.values = values_list
yield self._p.sendItem(
@ -546,9 +567,7 @@ class PubsubSchema(object):
form = data_form.findForm(item_elt, form_ns)
if form is None:
log.warning(
_(
"Can't parse previous item, update ignored: data form not found"
).format(reason=e)
_("Can't parse previous item, update ignored: data form not found")
)
else:
for name, field in form.fields.items():

View file

@ -64,7 +64,7 @@ class ImportPlugin(object):
@param name(unicode): import handler name
"""
assert name == name.lower().strip()
log.info(_("initializing {name} import handler").format(name=name))
log.info(_(f"initializing {name} import handler"))
import_handler.name = name
import_handler.register = partial(self.register, import_handler)
import_handler.unregister = partial(self.unregister, import_handler)
@ -139,16 +139,8 @@ class ImportPlugin(object):
else:
return importer.short_desc, importer.long_desc
def _doImport(
self,
import_handler,
name,
location,
options,
pubsub_service="",
pubsub_node="",
profile=C.PROF_KEY_NONE,
):
def _doImport(self, import_handler, name, location, options, pubsub_service="",
pubsub_node="", profile=C.PROF_KEY_NONE):
client = self.host.getClient(profile)
options = {key: str(value) for key, value in options.items()}
for option in import_handler.BOOL_OPTIONS:
@ -159,9 +151,11 @@ class ImportPlugin(object):
for option in import_handler.JSON_OPTIONS:
try:
options[option] = json.loads(options[option])
except KeyError:
pass
except ValueError:
raise exceptions.DataError(
_("invalid json option: {name}").format(name=option)
_(f"invalid json option: {option}")
)
pubsub_service = jid.JID(pubsub_service) if pubsub_service else None
return self.doImport(
@ -175,16 +169,8 @@ class ImportPlugin(object):
)
@defer.inlineCallbacks
def doImport(
self,
client,
import_handler,
name,
location,
options=None,
pubsub_service=None,
pubsub_node=None,
):
def doImport(self, client, import_handler, name, location, options=None,
pubsub_service=None, pubsub_node=None,):
"""Import data
@param import_handler(object): instance of the import handler
@ -193,7 +179,8 @@ class ImportPlugin(object):
can be an url, a file path, or anything which make sense
check importer description for more details
@param options(dict, None): extra options.
@param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported
@param pubsub_service(jid.JID, None): jid of the PubSub service where data must be
imported.
None to use profile's server
@param pubsub_node(unicode, None): PubSub node to use
None to use importer's default node

View file

@ -53,6 +53,7 @@ class MercurialProtocol(async_process.CommandProtocol):
@param path(unicode): path to the repository
@param command(unicode): hg command to run
@return D(bytes): stdout of the command
"""
assert "path" not in kwargs
kwargs["path"] = path
@ -86,8 +87,10 @@ class MercurialHandler(object):
return d
def export(self, repository):
return MercurialProtocol.run(repository, 'export', '-g', '-r', 'outgoing()',
'--encoding=utf-8')
d = MercurialProtocol.run(repository, 'export', '-g', '-r', 'outgoing()',
'--encoding=utf-8')
d.addCallback(lambda data: data.decode('utf-8'))
return d
def import_(self, repository, data, data_type, item_id, service, node, extra):
parsed_data = self.parse(data)
@ -97,7 +100,7 @@ class MercurialHandler(object):
except Exception:
parsed_name = ''
name = 'mr_{item_id}_{parsed_name}'.format(item_id=CLEAN_RE.sub('', item_id),
parsed_name=parsed_name)
parsed_name=parsed_name)
return MercurialProtocol.run(repository, 'qimport', '-g', '--name', name,
'--encoding=utf-8', '-', stdin=data)

View file

@ -254,7 +254,7 @@ class InvitationsPlugin(object):
try:
yield self.host.plugins['XEP-0077'].registerNewAccount(jid_,
password)
except error.StanzaError as e:
except error.StanzaError:
idx += 1
else:
break
@ -321,7 +321,7 @@ class InvitationsPlugin(object):
format_args['url'] = invite_url
yield sat_email.sendEmail(
self.host,
self.host.memory.config,
[email] + emails_extra,
(kwargs.pop('message_subject', None) or DEFAULT_SUBJECT).format(
**format_args),
@ -401,7 +401,7 @@ class InvitationsPlugin(object):
C.PROF_KEY_NONE: don't filter invitations
@return list(unicode): invitations uids
"""
invitations = yield list(self.invitations.items())
invitations = yield self.invitations.items()
if profile != C.PROF_KEY_NONE:
invitations = {id_:data for id_, data in invitations.items()
if data.get('profile') == profile}

View file

@ -106,8 +106,8 @@ class MergeRequests(object):
@param name(unicode): name of the handler
@param handler(object): instance of the handler.
It must have the following methods, which may all return a Deferred:
- check(repository): True if repository can be handled
- export(repository): return export data, i.e. the patches
- check(repository)->bool: True if repository can be handled
- export(repository)->str: return export data, i.e. the patches
- parse(export_data): parse report data and return a list of dict
(1 per patch) with:
- title: title of the commit message (first line)

View file

@ -348,9 +348,8 @@ class TextSyntaxes(object):
element.text = ''
return html.tostring(xhtml_elt, encoding=str, method="xml")
def convert(
self, text, syntax_from, syntax_to=_SYNTAX_XHTML, safe=True, profile=None
):
def convert(self, text, syntax_from, syntax_to=_SYNTAX_XHTML, safe=True,
profile=None):
"""Convert a text between two syntaxes
@param text: text to convert

View file

@ -20,11 +20,11 @@
from sat.core.i18n import _
from sat.core.constants import Const as C
from sat.core.log import getLogger
log = getLogger(__name__)
from twisted.words.xish import domish
from functools import partial
log = getLogger(__name__)
PLUGIN_INFO = {
C.PI_NAME: "Raw XML log Plugin",
C.PI_IMPORT_NAME: "XmlLog",
@ -74,9 +74,9 @@ class XmlLog(object):
def onSend(self, obj, client):
if isinstance(obj, str):
log = str(obj)
xml_log = obj
elif isinstance(obj, domish.Element):
log = obj.toXml()
xml_log = obj.toXml()
else:
log.error(_("INTERNAL ERROR: Unmanaged XML type"))
self.host.bridge.xmlLog("OUT", log, client.profile)
self.host.bridge.xmlLog("OUT", xml_log, client.profile)

View file

@ -108,7 +108,7 @@ class Context(potr.context.Context):
message data when an encrypted message is going to be sent
"""
assert isinstance(self.peer, jid.JID)
msg = msg_str
msg = msg_str.decode('utf-8')
client = self.user.client
log.debug("injecting encrypted message to {to}".format(to=self.peer))
if appdata is None:
@ -530,7 +530,7 @@ class OTR(object):
) # FIXME: temporary and unsecure, must be changed when frontends
# are refactored
otrctx = client._otr_context_manager.getContextForUser(to_jid)
query = otrctx.sendMessage(0, "?OTRv?")
query = otrctx.sendMessage(0, b"?OTRv?")
otrctx.inject(query)
def _otrSessionEnd(self, menu_data, profile):

View file

@ -241,12 +241,12 @@ class XEP_0048(object):
except KeyError:
pass
for url in urls_data:
for url, url_data in urls_data.items():
url_elt = storage_elt.addElement("url")
url_elt[XEP_0048.URL_KEY] = url
for attr in XEP_0048.URL_ATTRS:
try:
url_elt[attr] = url[attr]
url_elt[attr] = url_data[attr]
except KeyError:
pass
@ -440,7 +440,10 @@ class XEP_0048(object):
return ret
data = bookmarks_ori[type_]
for bookmark in data:
ret[_storage_location][bookmark.full()] = data[bookmark].copy()
if type_ == XEP_0048.MUC_TYPE:
ret[_storage_location][bookmark.full()] = data[bookmark].copy()
else:
ret[_storage_location][bookmark] = data[bookmark].copy()
return ret
for _storage_location in ("local", "private", "pubsub"):

View file

@ -44,7 +44,7 @@ except:
raise exceptions.MissingModule(
"Missing module pillow, please download/install it from https://python-pillow.github.io"
)
from io import StringIO
import io
try:
from twisted.words.protocols.xmlstream import XMPPHandler
@ -52,7 +52,8 @@ except ImportError:
from wokkel.subprotocols import XMPPHandler
AVATAR_PATH = "avatars"
AVATAR_DIM = (64, 64) #  FIXME: dim are not adapted to modern resolutions !
# AVATAR_DIM = (64, 64) #  FIXME: dim are not adapted to modern resolutions !
AVATAR_DIM = (128, 128)
IQ_GET = '/iq[@type="get"]'
NS_VCARD = "vcard-temp"
@ -318,7 +319,7 @@ class XEP_0054(object):
avatar_hash = yield threads.deferToThread(
self.savePhoto, client, elem, entity_jid
)
except (exceptions.DataError, exceptions.NotFound) as e:
except (exceptions.DataError, exceptions.NotFound):
avatar_hash = ""
vcard_dict["avatar"] = avatar_hash
except Exception as e:
@ -515,12 +516,13 @@ class XEP_0054(object):
left += offset
right -= offset
img = img.crop((left, upper, right, lower))
img_buf = StringIO()
img_buf = io.BytesIO()
img.save(img_buf, "PNG")
photo_elt = vcard_elt.addElement("PHOTO")
photo_elt.addElement("TYPE", content="image/png")
photo_elt.addElement("BINVAL", content=b64encode(img_buf.getvalue()))
image_b64 = b64encode(img_buf.getvalue()).decode('utf-8')
photo_elt.addElement("BINVAL", content=image_b64)
image_hash = sha1(img_buf.getvalue()).hexdigest()
with client.cache.cacheData(
PLUGIN_INFO["import_name"], image_hash, "image/png", MAX_AGE

View file

@ -175,17 +175,8 @@ ProxyInfos = namedtuple("ProxyInfos", ["host", "jid", "port"])
class Candidate(object):
def __init__(
self,
host,
port,
type_,
priority,
jid_,
id_=None,
priority_local=False,
factory=None,
):
def __init__(self, host, port, type_, priority, jid_, id_=None, priority_local=False,
factory=None,):
"""
@param host(unicode): host IP or domain
@param port(int): port
@ -227,15 +218,6 @@ class Candidate(object):
return self._priority
def __str__(self):
# similar to __unicode__ but we don't show jid and we encode id
return "Candidate ({0.priority}): host={0.host} port={0.port} type={0.type}{id}".format(
self,
id=" id={}".format(self.id if self.id is not None else "").encode(
"utf-8", "ignore"
),
)
def __unicode__(self):
return "Candidate ({0.priority}): host={0.host} port={0.port} jid={0.jid} type={0.type}{id}".format(
self, id=" id={}".format(self.id if self.id is not None else "")
)

Some files were not shown because too many files have changed in this diff Show more