core: various changes and fixes to work with new storage and D-Bus bridge:

- fixes coroutines handling in various places
- fixes types which are not serialised by Tx DBus
- XEP-0384: call storage methods in main thread in XEP: Python OMEMO's Promise use thread
  which prevent the use of AsyncIO loop. To work around that, callLater is used to launch
  storage method in main thread. This is a temporary workaround, as Python OMEMO should
  get rid of Promise implementation and threads soon.
This commit is contained in:
Goffi 2021-06-03 15:21:43 +02:00
commit 97df5f8be3

View file

@ -1131,12 +1131,12 @@ class Memory(object):
)
def asyncGetStringParamA(
self, name, category, attr="value", security_limit=C.NO_SECURITY_LIMIT,
self, name, category, attribute="value", security_limit=C.NO_SECURITY_LIMIT,
profile_key=C.PROF_KEY_NONE):
profile = self.getProfileName(profile_key)
return defer.ensureDeferred(self.params.asyncGetStringParamA(
name, category, attr, security_limit, profile
name, category, attribute, security_limit, profile
))
def _getParamsUI(self, security_limit, app, extra_s, profile_key):
@ -1170,20 +1170,22 @@ class Memory(object):
client = self.host.getClient(profile_key)
# we accept any type
data = data_format.deserialise(data_s, type_check=None)
return self.storage.setPrivateValue(
namespace, key, data, binary=True, profile=client.profile)
return defer.ensureDeferred(self.storage.setPrivateValue(
namespace, key, data, binary=True, profile=client.profile))
def _privateDataGet(self, namespace, key, profile_key):
client = self.host.getClient(profile_key)
d = self.storage.getPrivates(
namespace, [key], binary=True, profile=client.profile)
d = defer.ensureDeferred(
self.storage.getPrivates(
namespace, [key], binary=True, profile=client.profile)
)
d.addCallback(lambda data_dict: data_format.serialise(data_dict.get(key)))
return d
def _privateDataDelete(self, namespace, key, profile_key):
client = self.host.getClient(profile_key)
return self.storage.delPrivateValue(
namespace, key, binary=True, profile=client.profile)
return defer.ensureDeferred(self.storage.delPrivateValue(
namespace, key, binary=True, profile=client.profile))
## Files ##
@ -1251,8 +1253,7 @@ class Memory(object):
_("unknown access type: {type}").format(type=perm_type)
)
@defer.inlineCallbacks
def checkPermissionToRoot(self, client, file_data, peer_jid, perms_to_check):
async def checkPermissionToRoot(self, client, file_data, peer_jid, perms_to_check):
"""do checkFilePermission on file_data and all its parents until root"""
current = file_data
while True:
@ -1260,7 +1261,7 @@ class Memory(object):
parent = current["parent"]
if not parent:
break
files_data = yield self.getFiles(
files_data = await self.getFiles(
client, peer_jid=None, file_id=parent, perms_to_check=None
)
try:
@ -1268,8 +1269,7 @@ class Memory(object):
except IndexError:
raise exceptions.DataError("Missing parent")
@defer.inlineCallbacks
def _getParentDir(
async def _getParentDir(
self, client, path, parent, namespace, owner, peer_jid, perms_to_check
):
"""Retrieve parent node from a path, or last existing directory
@ -1293,7 +1293,7 @@ class Memory(object):
# non existing directories will be created
parent = ""
for idx, path_elt in enumerate(path_elts):
directories = yield self.storage.getFiles(
directories = await self.storage.getFiles(
client,
parent=parent,
type_=C.FILE_TYPE_DIRECTORY,
@ -1302,7 +1302,7 @@ class Memory(object):
owner=owner,
)
if not directories:
defer.returnValue((parent, path_elts[idx:]))
return (parent, path_elts[idx:])
# from this point, directories don't exist anymore, we have to create them
elif len(directories) > 1:
raise exceptions.InternalError(
@ -1312,7 +1312,7 @@ class Memory(object):
directory = directories[0]
self.checkFilePermission(directory, peer_jid, perms_to_check)
parent = directory["id"]
defer.returnValue((parent, []))
return (parent, [])
def getFileAffiliations(self, file_data: dict) -> Dict[jid.JID, str]:
"""Convert file access to pubsub like affiliations"""
@ -1484,8 +1484,7 @@ class Memory(object):
)
return peer_jid.userhostJID()
@defer.inlineCallbacks
def getFiles(
async def getFiles(
self, client, peer_jid, file_id=None, version=None, parent=None, path=None,
type_=None, file_hash=None, hash_algo=None, name=None, namespace=None,
mime_type=None, public_id=None, owner=None, access=None, projection=None,
@ -1536,7 +1535,7 @@ class Memory(object):
if path is not None:
path = str(path)
# permission are checked by _getParentDir
parent, remaining_path_elts = yield self._getParentDir(
parent, remaining_path_elts = await self._getParentDir(
client, path, parent, namespace, owner, peer_jid, perms_to_check
)
if remaining_path_elts:
@ -1546,16 +1545,16 @@ class Memory(object):
if parent and peer_jid:
# if parent is given directly and permission check is requested,
# we need to check all the parents
parent_data = yield self.storage.getFiles(client, file_id=parent)
parent_data = await self.storage.getFiles(client, file_id=parent)
try:
parent_data = parent_data[0]
except IndexError:
raise exceptions.DataError("mising parent")
yield self.checkPermissionToRoot(
await self.checkPermissionToRoot(
client, parent_data, peer_jid, perms_to_check
)
files = yield self.storage.getFiles(
files = await self.storage.getFiles(
client,
file_id=file_id,
version=version,
@ -1578,15 +1577,16 @@ class Memory(object):
to_remove = []
for file_data in files:
try:
self.checkFilePermission(file_data, peer_jid, perms_to_check, set_affiliation=True)
self.checkFilePermission(
file_data, peer_jid, perms_to_check, set_affiliation=True
)
except exceptions.PermissionError:
to_remove.append(file_data)
for file_data in to_remove:
files.remove(file_data)
defer.returnValue(files)
return files
@defer.inlineCallbacks
def setFile(
async def setFile(
self, client, name, file_id=None, version="", parent=None, path=None,
type_=C.FILE_TYPE_FILE, file_hash=None, hash_algo=None, size=None,
namespace=None, mime_type=None, public_id=None, created=None, modified=None,
@ -1668,13 +1668,13 @@ class Memory(object):
if path is not None:
path = str(path)
# _getParentDir will check permissions if peer_jid is set, so we use owner
parent, remaining_path_elts = yield self._getParentDir(
parent, remaining_path_elts = await self._getParentDir(
client, path, parent, namespace, owner, owner, perms_to_check
)
# if remaining directories don't exist, we have to create them
for new_dir in remaining_path_elts:
new_dir_id = shortuuid.uuid()
yield self.storage.setFile(
await self.storage.setFile(
client,
name=new_dir,
file_id=new_dir_id,
@ -1691,7 +1691,7 @@ class Memory(object):
elif parent is None:
parent = ""
yield self.storage.setFile(
await self.storage.setFile(
client,
file_id=file_id,
version=version,