XEP-0065,XEP-0096,XEP-0166,XEP-0235,XEP-0300: file transfer improvments:

huge patch sorry :)

many things are improved by this patch, notably:
- updated to last protocol changes (urn:xmpp:jingle:apps:file-transfer:5 and urn:xmpp:hashes:2)
- XEP-0234: file request implementation
- XEP-0234: methods to parse and generate <file> element (can be used by other plugins easily)
- XEP-0234: range data is now in a namedtuple
- path and namespace can be specified when sending/requesting a file (not standard, but needed for file sharing)
- better error/termination handling
- trigger points to handle file requests by other plugins
- preparation to use file plugins with components
This commit is contained in:
Goffi 2018-02-28 18:28:39 +01:00
commit 2e5991575f

View file

@ -34,14 +34,17 @@ from twisted.words.protocols.jabber.xmlstream import XMPPHandler
from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet import error as internet_error
from collections import namedtuple
import regex
NS_JINGLE_FT = 'urn:xmpp:jingle:apps:file-transfer:4'
NS_JINGLE_FT = 'urn:xmpp:jingle:apps:file-transfer:5'
PLUGIN_INFO = {
C.PI_NAME: "Jingle File Transfer",
C.PI_IMPORT_NAME: "XEP-0234",
C.PI_TYPE: "XEP",
C.PI_MODES: C.PLUG_MODE_BOTH,
C.PI_PROTOCOLS: ["XEP-0234"],
C.PI_DEPENDENCIES: ["XEP-0166", "XEP-0300", "FILE"],
C.PI_MAIN: "XEP_0234",
@ -49,6 +52,9 @@ PLUGIN_INFO = {
C.PI_DESCRIPTION: _("""Implementation of Jingle File Transfer""")
}
EXTRA_ALLOWED = {u'path', u'namespace', u'file_desc', u'file_hash'}
Range = namedtuple('Range', ('offset', 'length'))
class XEP_0234(object):
# TODO: assure everything is closed when file is sent or session terminate is received
@ -57,17 +63,19 @@ class XEP_0234(object):
def __init__(self, host):
log.info(_("plugin Jingle File Transfer initialization"))
self.host = host
host.registerNamespace('jingle-ft', NS_JINGLE_FT)
self._j = host.plugins["XEP-0166"] # shortcut to access jingle
self._j.registerApplication(NS_JINGLE_FT, self)
self._f = host.plugins["FILE"]
self._f.register(NS_JINGLE_FT, self.fileJingleSend, priority = 10000, method_name=u"Jingle")
self._hash = self.host.plugins["XEP-0300"]
host.bridge.addMethod("fileJingleSend", ".plugin", in_sign='sssss', out_sign='', method=self._fileJingleSend)
host.bridge.addMethod("fileJingleSend", ".plugin", in_sign='ssssa{ss}s', out_sign='', method=self._fileJingleSend, async=True)
host.bridge.addMethod("fileJingleRequest", ".plugin", in_sign='sssssa{ss}s', out_sign='s', method=self._fileJingleRequest, async=True)
def getHandler(self, client):
return XEP_0234_handler()
def _getProgressId(self, session, content_name):
def getProgressId(self, session, content_name):
"""Return a unique progress ID
@param session(dict): jingle session
@ -76,11 +84,154 @@ class XEP_0234(object):
"""
return u'{}_{}'.format(session['id'], content_name)
def _fileJingleSend(self, peer_jid, filepath, name="", file_desc="", profile=C.PROF_KEY_NONE):
client = self.host.getClient(profile)
return self.fileJingleSend(client, jid.JID(peer_jid), filepath, name or None, file_desc or None)
# generic methods
def fileJingleSend(self, client, peer_jid, filepath, name, file_desc=None):
def buildFileElement(self, name, file_hash=None, hash_algo=None, size=None, media_type=None, desc=None,
date=None, range_offset=None, range_length=None, path=None, namespace=None, file_elt=None):
"""Generate a <file> element with available metadata
@param file_hash(unicode, None): hash of the file
empty string to set <hash-used/> element
@param hash_algo(unicode, None): hash algorithm used
if file_hash is None and hash_algo is set, a <hash-used/> element will be generated
@param range_offset(int, None): offset where transfer must start
use -1 to add an empty <range/> element
@param date(int, unicode, None): date of last modification
0 to use current date
int to use an unix timestamp
else must be an unicode string which will be used as it (it must be an XMPP time)
@param file_elt(domish.Element, None): element to use
None to create a new one
@return (domish.Element): generated element
"""
if file_elt is None:
file_elt = domish.Element((NS_JINGLE_FT, u'file'))
for name, value in ((u'name', name), (u'size', size), ('media-type', media_type),
(u'desc', desc), (u'path', path), (u'namespace', namespace)):
if value is not None:
file_elt.addElement(name, content=unicode(value))
if date is not None:
if isinstance(date, int):
file_elt.addElement(u'date', utils.xmpp_date(date or None))
else:
file_elt.addElement(u'date', date)
if range_offset or range_length:
range_elt = file_elt.addElement(u'range')
if range_offset is not None and range_offset != -1:
range_elt[u'offset'] = range_offset
if range_length is not None:
range_elt[u'length'] = range_length
if file_hash is not None:
if not file_hash:
file_elt.addChild(self._hash.buildHashUsedElt())
else:
file_elt.addChild(self._hash.buildHashElt(file_hash, hash_algo))
elif hash_algo is not None:
file_elt.addChild(self._hash.buildHashUsedElt(hash_algo))
return file_elt
def buildFileElementFromDict(self, file_data, file_elt = None, **kwargs):
"""like buildFileElement but get values from a file_data dict
@param file_data(dict): metadata to use
@param **kwargs: data to override
"""
if kwargs:
file_data = file_data.copy()
file_data.update(kwargs)
(name, file_hash, hash_algo, size, media_type,
desc, date, path, namespace) = (file_data.get(u'name'),
file_data.get(u'file_hash'),
file_data.get(u'hash_algo'),
file_data.get(u'size'),
file_data.get(u'media-type'),
file_data.get(u'desc'),
file_data.get(u'date'),
file_data.get(u'path'),
file_data.get(u'namespace'))
try:
range_offset, range_length = file_data[u'range']
except KeyError:
range_offset = range_length = None
return self. buildFileElement(name = name, file_hash = file_hash, hash_algo = hash_algo, size = size,
media_type = media_type, desc = desc, date = date,
range_offset = range_offset, range_length = range_length, path = path,
namespace = namespace, file_elt = file_elt)
def parseFileElement(self, file_elt, file_data=None, given=False, parent_elt=None):
"""Parse a <file> element and file dictionary accordingly
@param file_data(dict, None): dict where the data will be set
following keys will be set (and overwritten if they already exist):
name, file_hash, hash_algo, size, media-type, desc, path, namespace, range
if None, a new dict is created
@param given(bool): if True, prefix hash key with "given_"
@param parent_elt(domish.Element, None): parent of the file element
if set, file_elt must not be set
@return (dict): file_data
@raise exceptions.NotFound: there is not <file> element in parent_elt
@raise exceptions.DataError: if file_elt uri is not NS_JINGLE_FT
"""
if parent_elt is not None:
if file_elt is not None:
raise exceptions.InternalError(u'file_elt must be None if parent_elt is set')
try:
file_elt = next(parent_elt.elements(NS_JINGLE_FT, u'file'))
except StopIteration:
raise exceptions.NotFound()
else:
if not file_elt or file_elt.uri != NS_JINGLE_FT:
raise exceptions.DataError(u'invalid <file> element: {stanza}'.format(stanza = file_elt.toXml()))
if file_data is None:
file_data = {}
for name in (u'name', u'media-type', u'desc', u'path', u'namespace'):
try:
file_data[name] = unicode(file_elt.elements(NS_JINGLE_FT, name).next())
except StopIteration:
pass
name = file_data.get(u'name')
if name == u'..':
# we don't want to go to parent dir when joining to a path
name = u'--'
file_data[u'name'] = name
elif name is not None and u'/' in name or u'\\' in name:
file_data[u'name'] = regex.pathEscape(name)
try:
file_data[u'size'] = int(unicode(file_elt.elements(NS_JINGLE_FT, u'size').next()))
except StopIteration:
pass
try:
range_elt = file_elt.elements(NS_JINGLE_FT, u'range').next()
except StopIteration:
pass
else:
offset = range_elt.getAttribute('offset')
length = range_elt.getAttribute('length')
file_data[u'range'] = Range(offset=offset, length=length)
prefix = u'given_' if given else u''
hash_algo_key, hash_key = u'hash_algo', prefix + u'file_hash'
try:
file_data[hash_algo_key], file_data[hash_key] = self._hash.parseHashElt(file_elt)
except exceptions.NotFound:
pass
return file_data
# bridge methods
def _fileJingleSend(self, peer_jid, filepath, name="", file_desc="", extra=None, profile=C.PROF_KEY_NONE):
client = self.host.getClient(profile)
return self.fileJingleSend(client, jid.JID(peer_jid), filepath, name or None, file_desc or None, extra or None)
@defer.inlineCallbacks
def fileJingleSend(self, client, peer_jid, filepath, name, file_desc=None, extra=None):
"""Send a file using jingle file transfer
@param peer_jid(jid.JID): destinee jid
@ -90,61 +241,160 @@ class XEP_0234(object):
@return (D(unicode)): progress id
"""
progress_id_d = defer.Deferred()
self._j.initiate(client,
peer_jid,
[{'app_ns': NS_JINGLE_FT,
'senders': self._j.ROLE_INITIATOR,
'app_kwargs': {'filepath': filepath,
'name': name,
'file_desc': file_desc,
'progress_id_d': progress_id_d},
}])
return progress_id_d
if extra is None:
extra = {}
if file_desc is not None:
extra['file_desc'] = file_desc
yield self._j.initiate(client,
peer_jid,
[{'app_ns': NS_JINGLE_FT,
'senders': self._j.ROLE_INITIATOR,
'app_kwargs': {'filepath': filepath,
'name': name,
'extra': extra,
'progress_id_d': progress_id_d},
}])
progress_id = yield progress_id_d
defer.returnValue(progress_id)
def _fileJingleRequest(self, peer_jid, filepath, name="", file_hash="", hash_algo="", extra=None, profile=C.PROF_KEY_NONE):
client = self.host.getClient(profile)
return self.fileJingleRequest(client, jid.JID(peer_jid), filepath, name or None, file_hash or None, hash_algo or None, extra or None)
@defer.inlineCallbacks
def fileJingleRequest(self, client, peer_jid, filepath, name=None, file_hash=None, hash_algo=None, extra=None):
"""Request a file using jingle file transfer
@param peer_jid(jid.JID): destinee jid
@param filepath(str): absolute path of the file
@param name(unicode, None): name of the file
@param file_hash(unicode, None): hash of the file
@return (D(unicode)): progress id
"""
progress_id_d = defer.Deferred()
if extra is None:
extra = {}
if file_hash is not None:
if hash_algo is None:
raise ValueError(_(u"hash_algo must be set if file_hash is set"))
extra['file_hash'] = file_hash
extra['hash_algo'] = hash_algo
else:
if hash_algo is not None:
raise ValueError(_(u"file_hash must be set if hash_algo is set"))
yield self._j.initiate(client,
peer_jid,
[{'app_ns': NS_JINGLE_FT,
'senders': self._j.ROLE_RESPONDER,
'app_kwargs': {'filepath': filepath,
'name': name,
'extra': extra,
'progress_id_d': progress_id_d},
}])
progress_id = yield progress_id_d
defer.returnValue(progress_id)
# jingle callbacks
def jingleSessionInit(self, client, session, content_name, filepath, name, file_desc, progress_id_d):
progress_id_d.callback(self._getProgressId(session, content_name))
def jingleSessionInit(self, client, session, content_name, filepath, name, extra, progress_id_d):
if extra is None:
extra = {}
else:
if not EXTRA_ALLOWED.issuperset(extra):
raise ValueError(_(u"only the following keys are allowed in extra: {keys}").format(
keys=u', '.join(EXTRA_ALLOWED)))
progress_id_d.callback(self.getProgressId(session, content_name))
content_data = session['contents'][content_name]
application_data = content_data['application_data']
assert 'file_path' not in application_data
application_data['file_path'] = filepath
file_data = application_data['file_data'] = {}
file_data['date'] = utils.xmpp_date()
file_data['desc'] = file_desc or ''
file_data['media-type'] = "application/octet-stream" # TODO
file_data['name'] = os.path.basename(filepath) if name is None else name
file_data['size'] = os.path.getsize(filepath)
desc_elt = domish.Element((NS_JINGLE_FT, 'description'))
file_elt = desc_elt.addElement("file")
for name in ('date', 'desc', 'media-type', 'name', 'size'):
file_elt.addElement(name, content=unicode(file_data[name]))
file_elt.addElement("range") # TODO
file_elt.addChild(self._hash.buildHashElt())
if content_data[u'senders'] == self._j.ROLE_INITIATOR:
# we send a file
file_data[u'date'] = utils.xmpp_date()
file_data[u'desc'] = extra.pop(u'file_desc', u'')
file_data[u'media-type'] = "application/octet-stream" # TODO
file_data[u'name'] = os.path.basename(filepath) if name is None else name
file_data[u'size'] = os.path.getsize(filepath)
if u'namespace' in extra:
file_data[u'namespace'] = extra[u'namespace']
if u'path' in extra:
file_data[u'path'] = extra[u'path']
self.buildFileElementFromDict(file_data, file_elt=file_elt, file_hash=u'')
file_elt.addElement("range") # TODO
else:
# we request a file
file_hash = extra.pop(u'file_hash', u'')
if not name and not file_hash:
raise ValueError(_(u'you need to provide at least name or file hash'))
if name:
file_data[u'name'] = name
if file_hash:
file_data[u'file_hash'] = file_hash
file_data[u'hash_algo'] = extra[u'hash_algo']
else:
file_data[u'hash_algo'] = self._hash.getDefaultAlgo()
if u'namespace' in extra:
file_data[u'namespace'] = extra[u'namespace']
if u'path' in extra:
file_data[u'path'] = extra[u'path']
self.buildFileElementFromDict(file_data, file_elt=file_elt)
return desc_elt
def jingleRequestConfirmation(self, client, action, session, content_name, desc_elt):
"""This method request confirmation for a jingle session"""
content_data = session['contents'][content_name]
if content_data['senders'] not in (self._j.ROLE_INITIATOR, self._j.ROLE_RESPONDER):
senders = content_data[u'senders']
if senders not in (self._j.ROLE_INITIATOR, self._j.ROLE_RESPONDER):
log.warning(u"Bad sender, assuming initiator")
content_data['senders'] = self._j.ROLE_INITIATOR
senders = content_data[u'senders'] = self._j.ROLE_INITIATOR
# first we grab file informations
try:
file_elt = desc_elt.elements(NS_JINGLE_FT, 'file').next()
except StopIteration:
raise failure.Failure(exceptions.DataError)
file_data = {'progress_id': self._getProgressId(session, content_name)}
for name in ('date', 'desc', 'media-type', 'name', 'range', 'size'):
try:
file_data[name] = unicode(file_elt.elements(NS_JINGLE_FT, name).next())
except StopIteration:
file_data[name] = ''
file_data = {'progress_id': self.getProgressId(session, content_name)}
if senders == self._j.ROLE_RESPONDER:
# we send the file
return self._fileSendingRequestConf(client, session, content_data, content_name, file_data, file_elt)
else:
# we receive the file
return self._fileReceivingRequestConf(client, session, content_data, content_name, file_data, file_elt)
@defer.inlineCallbacks
def _fileSendingRequestConf(self, client, session, content_data, content_name, file_data, file_elt):
"""parse file_elt, and handle file retrieving/permission checking"""
self.parseFileElement(file_elt, file_data)
content_data['application_data']['file_data'] = file_data
finished_d = content_data['finished_d'] = defer.Deferred()
# confirmed_d is a deferred returning confimed value (only used if cont is False)
cont, confirmed_d = self.host.trigger.returnPoint("XEP-0234_fileSendingRequest", client, session, content_data, content_name, file_data, file_elt)
if not cont:
confirmed = yield confirmed_d
if confirmed:
args = [client, session, content_name, content_data]
finished_d.addCallbacks(self._finishedCb, self._finishedEb, args, None, args)
defer.returnValue(confirmed)
log.warning(_(u'File continue is not implemented yet'))
defer.returnValue(False)
def _fileReceivingRequestConf(self, client, session, content_data, content_name, file_data, file_elt):
"""parse file_elt, and handle user permission/file opening"""
self.parseFileElement(file_elt, file_data, given=True)
try:
hash_algo, file_data['hash_given'] = self._hash.parseHashElt(file_elt)
hash_algo, file_data['given_file_hash'] = self._hash.parseHashElt(file_elt)
except exceptions.NotFound:
raise failure.Failure(exceptions.DataError)
try:
hash_algo = self._hash.parseHashUsedElt(file_elt)
except exceptions.NotFound:
raise failure.Failure(exceptions.DataError)
if hash_algo is not None:
file_data['hash_algo'] = hash_algo
@ -166,11 +416,12 @@ class XEP_0234(object):
# now we actualy request permission to user
def gotConfirmation(confirmed):
if confirmed:
finished_d = content_data['finished_d'] = defer.Deferred()
args = [client, session, content_name, content_data]
finished_d.addCallbacks(self._finishedCb, self._finishedEb, args, None, args)
return confirmed
# deferred to track end of transfer
finished_d = content_data['finished_d'] = defer.Deferred()
d = self._f.getDestDir(client, session['peer_jid'], content_data, file_data, stream_object=True)
d.addCallback(gotConfirmation)
return d
@ -186,23 +437,47 @@ class XEP_0234(object):
file_elt.elements(NS_JINGLE_FT, 'range').next()
except StopIteration:
# initiator doesn't manage <range>, but we do so we advertise it
# FIXME: to be checked
log.debug("adding <range> element")
file_elt.addElement('range')
elif action == self._j.A_SESSION_ACCEPT:
assert not 'stream_object' in content_data
file_data = application_data['file_data']
file_path = application_data['file_path']
size = file_data['size']
# XXX: hash security is not critical here, so we just take the higher mandatory one
hasher = file_data['hash_hasher'] = self._hash.getHasher('sha-256')
content_data['stream_object'] = stream.FileStreamObject(
self.host,
client,
file_path,
uid=self._getProgressId(session, content_name),
size=size,
data_cb=lambda data: hasher.update(data),
)
senders = content_data[u'senders']
if senders != session[u'role']:
# we are receiving the file
try:
# did the responder specified the size of the file?
file_elt = next(desc_elt.elements(NS_JINGLE_FT, u'file'))
size_elt = next(file_elt.elements(NS_JINGLE_FT, u'size'))
size = int(unicode(size_elt))
except (StopIteration, ValueError):
size = None
# XXX: hash security is not critical here, so we just take the higher mandatory one
hasher = file_data['hash_hasher'] = self._hash.getHasher()
content_data['stream_object'] = stream.FileStreamObject(
self.host,
client,
file_path,
mode='wb',
uid=self.getProgressId(session, content_name),
size=size,
data_cb=lambda data: hasher.update(data),
)
else:
# we are sending the file
size = file_data['size']
# XXX: hash security is not critical here, so we just take the higher mandatory one
hasher = file_data['hash_hasher'] = self._hash.getHasher()
content_data['stream_object'] = stream.FileStreamObject(
self.host,
client,
file_path,
uid=self.getProgressId(session, content_name),
size=size,
data_cb=lambda data: hasher.update(data),
)
finished_d = content_data['finished_d'] = defer.Deferred()
args = [client, session, content_name, content_data]
finished_d.addCallbacks(self._finishedCb, self._finishedEb, args, None, args)
@ -237,15 +512,21 @@ class XEP_0234(object):
file_elt = elt.elements((NS_JINGLE_FT, 'file')).next()
except StopIteration:
raise exceptions.DataError
algo, file_data['hash_given'] = self._hash.parseHashElt(file_elt)
algo, file_data['given_file_hash'] = self._hash.parseHashElt(file_elt)
if algo != file_data.get('hash_algo'):
log.warning(u"Hash algorithm used in given hash ({peer_algo}) doesn't correspond to the one we have used ({our_algo})"
.format(peer_algo=algo, our_algo=file_data.get('hash_algo')))
log.warning(u"Hash algorithm used in given hash ({peer_algo}) doesn't correspond to the one we have used ({our_algo}) [{profile}]"
.format(peer_algo=algo, our_algo=file_data.get('hash_algo'), profile=client.profile))
else:
self._receiverTryTerminate(client, session, content_name, content_data)
else:
raise NotImplementedError
def jingleTerminate(self, client, action, session, content_name, jingle_elt):
if jingle_elt.decline:
# progress is the only way to tell to frontends that session has been declined
progress_id = self.getProgressId(session, content_name)
self.host.bridge.progressError(progress_id, C.PROGRESS_ERROR_DECLINED, client.profile)
def _sendCheckSum(self, client, session, content_name, content_data):
"""Send the session-info with the hash checksum"""
file_data = content_data['application_data']['file_data']
@ -272,10 +553,10 @@ class XEP_0234(object):
if not content_data.get('transfer_finished', False):
return False
file_data = content_data['application_data']['file_data']
hash_given = file_data.get('hash_given')
if hash_given is None:
given_hash = file_data.get('given_file_hash')
if given_hash is None:
if last_try:
log.warning(u"sender didn't sent hash checksum, we can't check the file")
log.warning(u"sender didn't sent hash checksum, we can't check the file [{profile}]".format(profile=client.profile))
self._j.delayedContentTerminate(client, session, content_name)
content_data['stream_object'].close()
return True
@ -283,7 +564,7 @@ class XEP_0234(object):
hasher = file_data['hash_hasher']
hash_ = hasher.hexdigest()
if hash_ == hash_given:
if hash_ == given_hash:
log.info(u"Hash checked, file was successfully transfered: {}".format(hash_))
progress_metadata = {'hash': hash_,
'hash_algo': file_data['hash_algo'],
@ -295,7 +576,7 @@ class XEP_0234(object):
progress_metadata=None
error = u"Hash mismatch: given={algo}:{given}, calculated={algo}:{our}".format(
algo = file_data['hash_algo'],
given = hash_given,
given = given_hash,
our = hash_)
self._j.delayedContentTerminate(client, session, content_name)