diff --git a/sat/plugins/plugin_misc_upload.py b/sat/plugins/plugin_misc_upload.py
index 2ede5823..199b8427 100644
--- a/sat/plugins/plugin_misc_upload.py
+++ b/sat/plugins/plugin_misc_upload.py
@@ -1,7 +1,6 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# SAT plugin for file tansfer
+# SAT plugin for uploading files
# Copyright (C) 2009-2019 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
@@ -17,18 +16,19 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
-from sat.core.i18n import _, D_
-from sat.core.constants import Const as C
-from sat.core.log import getLogger
-
-log = getLogger(__name__)
-from sat.core import exceptions
-from sat.tools import xml_tools
+import os
+import os.path
from twisted.internet import defer
from twisted.words.protocols.jabber import jid
from twisted.words.protocols.jabber import error as jabber_error
-import os
-import os.path
+from sat.core.i18n import _, D_
+from sat.core.constants import Const as C
+from sat.tools.common import data_format
+from sat.core.log import getLogger
+from sat.core import exceptions
+from sat.tools import xml_tools
+
+log = getLogger(__name__)
PLUGIN_INFO = {
@@ -43,7 +43,6 @@ PLUGIN_INFO = {
UPLOADING = D_("Please select a file to upload")
UPLOADING_TITLE = D_("File upload")
-BOOL_OPTIONS = ("ignore_tls_errors",)
class UploadPlugin(object):
@@ -55,7 +54,7 @@ class UploadPlugin(object):
host.bridge.addMethod(
"fileUpload",
".plugin",
- in_sign="sssa{ss}s",
+ in_sign="sssss",
out_sign="a{ss}",
method=self._fileUpload,
async_=True,
@@ -63,24 +62,17 @@ class UploadPlugin(object):
self._upload_callbacks = []
def _fileUpload(
- self, filepath, filename, upload_jid_s="", options=None, profile=C.PROF_KEY_NONE
+ self, filepath, filename, upload_jid_s="", options='', profile=C.PROF_KEY_NONE
):
client = self.host.getClient(profile)
upload_jid = jid.JID(upload_jid_s) if upload_jid_s else None
- if options is None:
- options = {}
- # we convert values that are well-known booleans
- for bool_option in BOOL_OPTIONS:
- try:
- options[bool_option] = C.bool(options[bool_option])
- except KeyError:
- pass
+ options = data_format.deserialise(options)
- return self.fileUpload(
- client, filepath, filename or None, upload_jid, options or None
- )
+ return defer.ensureDeferred(self.fileUpload(
+ client, filepath, filename or None, upload_jid, options
+ ))
- def fileUpload(self, client, filepath, filename, upload_jid, options):
+ async def fileUpload(self, client, filepath, filename, upload_jid, options):
"""Send a file using best available method
parameters are the same as for [upload]
@@ -88,16 +80,15 @@ class UploadPlugin(object):
message
"""
- def uploadCb(data):
- progress_id, __ = data
- return {"progress": progress_id}
-
- def uploadEb(fail):
- if (isinstance(fail.value, jabber_error.StanzaError)
- and fail.value.condition == 'not-acceptable'):
- reason = fail.value.text
+ try:
+ progress_id, __ = await self.upload(
+ client, filepath, filename, upload_jid, options)
+ except Exception as e:
+ if (isinstance(e, jabber_error.StanzaError)
+ and e.condition == 'not-acceptable'):
+ reason = e.text
else:
- reason = str(fail.value)
+ reason = str(e)
msg = D_("Can't upload file: {reason}").format(reason=reason)
log.warning(msg)
return {
@@ -105,14 +96,11 @@ class UploadPlugin(object):
msg, D_("Can't upload file"), C.XMLUI_DATA_LVL_WARNING
).toXml()
}
+ else:
+ return {"progress": progress_id}
- d = self.upload(client, filepath, filename, upload_jid, options)
- d.addCallback(uploadCb)
- d.addErrback(uploadEb)
- return d
-
- @defer.inlineCallbacks
- def upload(self, client, filepath, filename=None, upload_jid=None, options=None):
+ async def upload(self, client, filepath, filename=None, upload_jid=None,
+ options=None):
"""Send a file using best available method
@param filepath(str): absolute path to the file
@@ -133,18 +121,17 @@ class UploadPlugin(object):
raise exceptions.DataError("The given path doesn't link to a file")
for method_name, available_cb, upload_cb, priority in self._upload_callbacks:
try:
- upload_jid = yield available_cb(upload_jid, client.profile)
+ upload_jid = await available_cb(client, upload_jid)
except exceptions.NotFound:
continue # no entity managing this extension found
log.info(
"{name} method will be used to upload the file".format(name=method_name)
)
- progress_id_d, download_d = yield upload_cb(
- filepath, filename, upload_jid, options, client.profile
+ progress_id, download_d = await upload_cb(
+ client, filepath, filename, upload_jid, options
)
- progress_id = yield progress_id_d
- defer.returnValue((progress_id, download_d))
+ return progress_id, download_d
raise exceptions.NotFound("Can't find any method to upload a file")
diff --git a/sat/plugins/plugin_xep_0363.py b/sat/plugins/plugin_xep_0363.py
index a491bbe1..746c672e 100644
--- a/sat/plugins/plugin_xep_0363.py
+++ b/sat/plugins/plugin_xep_0363.py
@@ -17,30 +17,26 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
-from sat.core.i18n import _
-from sat.core.constants import Const as C
-from sat.core.log import getLogger
-
-log = getLogger(__name__)
-from sat.core import exceptions
+import os.path
+import mimetypes
+from dataclasses import dataclass
from wokkel import disco, iwokkel
from zope.interface import implementer
from twisted.words.protocols.jabber import jid
from twisted.words.protocols.jabber.xmlstream import XMPPHandler
from twisted.internet import reactor
from twisted.internet import defer
-from twisted.internet import ssl
-from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
from twisted.web import client as http_client
from twisted.web import http_headers
-from twisted.web import iweb
-from twisted.python import failure
-from collections import namedtuple
-from OpenSSL import SSL
-import os.path
-import mimetypes
+from sat.core.i18n import _
+from sat.core.constants import Const as C
+from sat.core.log import getLogger
+from sat.core import exceptions
+from sat.tools import web as sat_web
+log = getLogger(__name__)
+
PLUGIN_INFO = {
C.PI_NAME: "HTTP File Upload",
C.PI_IMPORT_NAME: "XEP-0363",
@@ -56,37 +52,12 @@ NS_HTTP_UPLOAD = "urn:xmpp:http:upload:0"
ALLOWED_HEADERS = ('authorization', 'cookie', 'expires')
-Slot = namedtuple("Slot", ["put", "get", "headers"])
-
-
-@implementer(IOpenSSLClientConnectionCreator)
-class NoCheckConnectionCreator(object):
- def __init__(self, hostname, ctx):
- self._ctx = ctx
-
- def clientConnectionForTLS(self, tlsProtocol):
- context = self._ctx
- connection = SSL.Connection(context, None)
- connection.set_app_data(tlsProtocol)
- return connection
-
-
-@implementer(iweb.IPolicyForHTTPS)
-class NoCheckContextFactory(ssl.ClientContextFactory):
- """Context factory which doesn't do TLS certificate check
-
- /!\\ it's obvisously a security flaw to use this class,
- and it should be used only with explicite agreement from the end used
- """
-
- def creatorForNetloc(self, hostname, port):
- log.warning(
- "TLS check disabled for {host} on port {port}".format(
- host=hostname, port=port
- )
- )
- certificateOptions = ssl.CertificateOptions(trustRoot=None)
- return NoCheckConnectionCreator(hostname, certificateOptions.getContext())
+@dataclass
+class Slot:
+ """Upload slot"""
+ put: str
+ get: str
+ headers: list
class XEP_0363(object):
@@ -115,8 +86,7 @@ class XEP_0363(object):
def getHandler(self, client):
return XEP_0363_handler()
- @defer.inlineCallbacks
- def getHTTPUploadEntity(self, upload_jid=None, profile=C.PROF_KEY_NONE):
+ async def getHTTPUploadEntity(self, client, upload_jid=None):
"""Get HTTP upload capable entity
upload_jid is checked, then its components
@@ -124,35 +94,35 @@ class XEP_0363(object):
@return(D(jid.JID)): first HTTP upload capable entity
@raise exceptions.NotFound: no entity found
"""
- client = self.host.getClient(profile)
try:
entity = client.http_upload_service
except AttributeError:
- found_entities = yield self.host.findFeaturesSet(client, (NS_HTTP_UPLOAD,))
+ found_entities = await self.host.findFeaturesSet(client, (NS_HTTP_UPLOAD,))
try:
entity = client.http_upload_service = next(iter(found_entities))
except StopIteration:
entity = client.http_upload_service = None
if entity is None:
- raise failure.Failure(exceptions.NotFound("No HTTP upload entity found"))
+ raise exceptions.NotFound("No HTTP upload entity found")
- defer.returnValue(entity)
+ return entity
def _fileHTTPUpload(self, filepath, filename="", upload_jid="",
ignore_tls_errors=False, profile=C.PROF_KEY_NONE):
assert os.path.isabs(filepath) and os.path.isfile(filepath)
- progress_id_d, __ = self.fileHTTPUpload(
+ client = self.host.getClient(profile)
+ progress_id_d, __ = defer.ensureDeferred(self.fileHTTPUpload(
+ client,
filepath,
filename or None,
jid.JID(upload_jid) if upload_jid else None,
{"ignore_tls_errors": ignore_tls_errors},
- profile,
- )
+ ))
return progress_id_d
- def fileHTTPUpload(self, filepath, filename=None, upload_jid=None, options=None,
- profile=C.PROF_KEY_NONE):
+ async def fileHTTPUpload(
+ self, client, filepath, filename=None, upload_jid=None, options=None):
"""Upload a file through HTTP
@param filepath(str): absolute path of the file
@@ -169,137 +139,96 @@ class XEP_0363(object):
if options is None:
options = {}
ignore_tls_errors = options.get("ignore_tls_errors", False)
- client = self.host.getClient(profile)
filename = filename or os.path.basename(filepath)
size = os.path.getsize(filepath)
- progress_id_d = defer.Deferred()
- download_d = defer.Deferred()
- d = self.getSlot(client, filename, size, upload_jid=upload_jid)
- d.addCallbacks(
- self._getSlotCb,
- self._getSlotEb,
- (client, progress_id_d, download_d, filepath, size, ignore_tls_errors),
- None,
- (client, progress_id_d, download_d),
- )
- return progress_id_d, download_d
- def _getSlotEb(self, fail, client, progress_id_d, download_d):
- """an error happened while trying to get slot"""
- log.warning("Can't get upload slot: {reason}".format(reason=fail.value))
- progress_id_d.errback(fail)
- download_d.errback(fail)
- def _getSlotCb(self, slot, client, progress_id_d, download_d, path, size,
- ignore_tls_errors=False):
- """Called when slot is received, try to do the upload
-
- @param slot(Slot): slot instance with the get and put urls
- @param progress_id_d(defer.Deferred): Deferred to call when progress_id is known
- @param progress_id_d(defer.Deferred): Deferred to call with URL when upload is
- done
- @param path(str): path to the file to upload
- @param size(int): size of the file to upload
- @param ignore_tls_errors(bool): ignore TLS certificate is True
- @return (tuple
- """
- log.debug(f"Got upload slot: {slot}")
- sat_file = self.host.plugins["FILE"].File(
- self.host, client, path, size=size, auto_end_signals=False
- )
- progress_id_d.callback(sat_file.uid)
- file_producer = http_client.FileBodyProducer(sat_file)
- if ignore_tls_errors:
- agent = http_client.Agent(reactor, NoCheckContextFactory())
+ size_adjust = []
+ #: this trigger can be used to modify the requested size, it is notably useful
+ #: with encryption. The size_adjust is a list which can be filled by int to add
+ #: to the initial size
+ self.host.trigger.point(
+ "XEP-0363_upload_size", client, options, filepath, size, size_adjust,
+ triggers_no_cancel=True)
+ if size_adjust:
+ size = sum([size, *size_adjust])
+ try:
+ slot = await self.getSlot(client, filename, size, upload_jid=upload_jid)
+ except Exception as e:
+ log.warning(_("Can't get upload slot: {reason}").format(reason=e))
+ raise e
else:
- agent = http_client.Agent(reactor)
+ log.debug(f"Got upload slot: {slot}")
+ sat_file = self.host.plugins["FILE"].File(
+ self.host, client, filepath, size=size, auto_end_signals=False
+ )
+ progress_id = sat_file.uid
- headers = {"User-Agent": [C.APP_NAME.encode("utf-8")]}
- for name, value in slot.headers:
- name = name.encode('utf-8')
- value = value.encode('utf-8')
- headers[name] = value
+ file_producer = http_client.FileBodyProducer(sat_file)
- d = agent.request(
- b"PUT",
- slot.put.encode("utf-8"),
- http_headers.Headers(headers),
- file_producer,
- )
- d.addCallbacks(
- self._uploadCb,
- self._uploadEb,
- (sat_file, slot, download_d),
- None,
- (sat_file, download_d),
- )
- return d
+ if ignore_tls_errors:
+ agent = http_client.Agent(reactor, sat_web.NoCheckContextFactory())
+ else:
+ agent = http_client.Agent(reactor)
- def _uploadCb(self, __, sat_file, slot, download_d):
+ headers = {"User-Agent": [C.APP_NAME.encode("utf-8")]}
+
+ for name, value in slot.headers:
+ name = name.encode('utf-8')
+ value = value.encode('utf-8')
+ headers[name] = value
+
+
+ await self.host.trigger.asyncPoint(
+ "XEP-0363_upload", client, options, sat_file, file_producer, slot,
+ triggers_no_cancel=True)
+
+ download_d = agent.request(
+ b"PUT",
+ slot.put.encode("utf-8"),
+ http_headers.Headers(headers),
+ file_producer,
+ )
+ download_d.addCallbacks(
+ self._uploadCb,
+ self._uploadEb,
+ (sat_file, slot),
+ None,
+ (sat_file),
+ )
+
+ return progress_id, download_d
+
+ def _uploadCb(self, __, sat_file, slot):
"""Called once file is successfully uploaded
@param sat_file(SatFile): file used for the upload
- should be closed, be is needed to send the progressFinished signal
+ should be closed, but it is needed to send the progressFinished signal
@param slot(Slot): put/get urls
"""
log.info("HTTP upload finished")
sat_file.progressFinished({"url": slot.get})
- download_d.callback(slot.get)
+ return slot.get
- def _uploadEb(self, fail, sat_file, download_d):
+ def _uploadEb(self, failure_, sat_file):
"""Called on unsuccessful upload
@param sat_file(SatFile): file used for the upload
should be closed, be is needed to send the progressError signal
"""
- download_d.errback(fail)
try:
- wrapped_fail = fail.value.reasons[0]
+ wrapped_fail = failure_.value.reasons[0]
except (AttributeError, IndexError) as e:
log.warning(_("upload failed: {reason}").format(reason=e))
- sat_file.progressError(str(fail))
- raise fail
+ sat_file.progressError(str(failure_))
else:
- if wrapped_fail.check(SSL.Error):
+ if wrapped_fail.check(sat_web.SSLError):
msg = "TLS validation error, can't connect to HTTPS server"
else:
msg = "can't upload file"
log.warning(msg + ": " + str(wrapped_fail.value))
sat_file.progressError(msg)
-
- def _gotSlot(self, iq_elt, client):
- """Slot have been received
-
- This method convert the iq_elt result to a Slot instance
- @param iq_elt(domish.Element): result as specified in XEP-0363
- """
- try:
- slot_elt = next(iq_elt.elements(NS_HTTP_UPLOAD, "slot"))
- put_elt = next(slot_elt.elements(NS_HTTP_UPLOAD, "put"))
- put_url = put_elt['url']
- get_elt = next(slot_elt.elements(NS_HTTP_UPLOAD, "get"))
- get_url = get_elt['url']
- except (StopIteration, KeyError):
- raise exceptions.DataError("Incorrect stanza received from server")
- headers = []
- for header_elt in put_elt.elements(NS_HTTP_UPLOAD, "header"):
- try:
- name = header_elt["name"]
- value = str(header_elt)
- except KeyError:
- log.warning(_("Invalid header element: {xml}").format(
- iq_elt.toXml()))
- continue
- name = name.replace('\n', '')
- value = value.replace('\n', '')
- if name.lower() not in ALLOWED_HEADERS:
- log.warning(_('Ignoring unauthorised header "{name}": {xml}')
- .format(name=name, xml = iq_elt.toXml()))
- continue
- headers.append((name, value))
-
- slot = Slot(put=put_url, get=get_url, headers=tuple(headers))
- return slot
+ raise failure_
def _getSlot(self, filename, size, content_type, upload_jid,
profile_key=C.PROF_KEY_NONE):
@@ -312,13 +241,13 @@ class XEP_0363(object):
@param content_type(unicode, None): MIME type of the content
empty string or None to guess automatically
"""
- filename = filename.replace("/", "_")
client = self.host.getClient(profile_key)
- return self.getSlot(
+ filename = filename.replace("/", "_")
+ return defer.ensureDeferred(self.getSlot(
client, filename, size, content_type or None, upload_jid or None
- )
+ ))
- def getSlot(self, client, filename, size, content_type=None, upload_jid=None):
+ async def getSlot(self, client, filename, size, content_type=None, upload_jid=None):
"""Get a slot (i.e. download/upload links)
@param filename(unicode): name to use for the upload
@@ -340,18 +269,12 @@ class XEP_0363(object):
try:
upload_jid = client.http_upload_service
except AttributeError:
- d = self.getHTTPUploadEntity(profile=client.profile)
- d.addCallback(
- lambda found_entity: self.getSlot(
- client, filename, size, content_type, found_entity
- )
- )
- return d
+ found_entity = await self.getHTTPUploadEntity(profile=client.profile)
+ return await self.getSlot(
+ client, filename, size, content_type, found_entity)
else:
if upload_jid is None:
- raise failure.Failure(
- exceptions.NotFound("No HTTP upload entity found")
- )
+ raise exceptions.NotFound("No HTTP upload entity found")
iq_elt = client.IQ("get")
iq_elt["to"] = upload_jid.full()
@@ -361,10 +284,35 @@ class XEP_0363(object):
if content_type is not None:
request_elt["content-type"] = content_type
- d = iq_elt.send()
- d.addCallback(self._gotSlot, client)
+ iq_result_elt = await iq_elt.send()
- return d
+ try:
+ slot_elt = next(iq_result_elt.elements(NS_HTTP_UPLOAD, "slot"))
+ put_elt = next(slot_elt.elements(NS_HTTP_UPLOAD, "put"))
+ put_url = put_elt['url']
+ get_elt = next(slot_elt.elements(NS_HTTP_UPLOAD, "get"))
+ get_url = get_elt['url']
+ except (StopIteration, KeyError):
+ raise exceptions.DataError("Incorrect stanza received from server")
+
+ headers = []
+ for header_elt in put_elt.elements(NS_HTTP_UPLOAD, "header"):
+ try:
+ name = header_elt["name"]
+ value = str(header_elt)
+ except KeyError:
+ log.warning(_("Invalid header element: {xml}").format(
+ iq_result_elt.toXml()))
+ continue
+ name = name.replace('\n', '')
+ value = value.replace('\n', '')
+ if name.lower() not in ALLOWED_HEADERS:
+ log.warning(_('Ignoring unauthorised header "{name}": {xml}')
+ .format(name=name, xml = iq_result_elt.toXml()))
+ continue
+ headers.append((name, value))
+
+ return Slot(put=put_url, get=get_url, headers=headers)
@implementer(iwokkel.IDisco)
diff --git a/sat/tools/web.py b/sat/tools/web.py
new file mode 100644
index 00000000..e0aaf112
--- /dev/null
+++ b/sat/tools/web.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+# SàT: an XMPP client
+# Copyright (C) 2009-2019 Jérôme Poisson (goffi@goffi.org)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+from OpenSSL import SSL
+from zope.interface import implementer
+from treq.client import HTTPClient
+from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
+from twisted.internet import reactor, ssl
+from twisted.web import iweb
+from twisted.web import client as http_client
+from sat.core.log import getLogger
+
+
+log = getLogger(__name__)
+
+
+SSLError = SSL.Error
+
+
+@implementer(IOpenSSLClientConnectionCreator)
+class NoCheckConnectionCreator(object):
+ def __init__(self, hostname, ctx):
+ self._ctx = ctx
+
+ def clientConnectionForTLS(self, tlsProtocol):
+ context = self._ctx
+ connection = SSL.Connection(context, None)
+ connection.set_app_data(tlsProtocol)
+ return connection
+
+
+@implementer(iweb.IPolicyForHTTPS)
+class NoCheckContextFactory(ssl.ClientContextFactory):
+ """Context factory which doesn't do TLS certificate check
+
+ /!\\ it's obvisously a security flaw to use this class,
+ and it should be used only with explicit agreement from the end used
+ """
+
+ def creatorForNetloc(self, hostname, port):
+ log.warning(
+ "TLS check disabled for {host} on port {port}".format(
+ host=hostname, port=port
+ )
+ )
+ certificateOptions = ssl.CertificateOptions(trustRoot=None)
+ return NoCheckConnectionCreator(hostname, certificateOptions.getContext())
+
+
+#: following treq doesn't check TLS, obviously it is unsecure and should not be used
+#: without explicit warning
+treq_no_ssl = HTTPClient(http_client.Agent(reactor, NoCheckContextFactory))
diff --git a/sat_frontends/jp/cmd_file.py b/sat_frontends/jp/cmd_file.py
index d5bc5934..305bc13c 100644
--- a/sat_frontends/jp/cmd_file.py
+++ b/sat_frontends/jp/cmd_file.py
@@ -478,7 +478,7 @@ class Upload(base.CommandBase):
options = {}
if self.args.ignore_tls_errors:
- options["ignore_tls_errors"] = C.BOOL_TRUE
+ options["ignore_tls_errors"] = True
path = os.path.abspath(file_)
try:
@@ -486,7 +486,7 @@ class Upload(base.CommandBase):
path,
"",
self.full_dest_jid,
- options,
+ data_format.serialise(options),
self.profile,
)
except Exception as e:
diff --git a/setup.py b/setup.py
index b91fc93b..b19c7584 100644
--- a/setup.py
+++ b/setup.py
@@ -45,6 +45,7 @@ install_requires = [
'sat_tmp >= 0.7.0a4',
'shortuuid',
'twisted[tls] >= 19.7.0',
+ 'treq',
'urwid >= 1.2.0',
'urwid-satext >= 0.7.0a2',
'wokkel >= 0.7.1',