Reformatted code base using black.
This commit is contained in:
parent
56319cbad3
commit
53fdaa9d09
1 changed files with 12604 additions and 8966 deletions
|
|
@ -72,18 +72,20 @@ class DownloadPlugin(object):
|
|||
)
|
||||
self._download_callbacks = {}
|
||||
self._scheme_callbacks = {}
|
||||
self.register_scheme('http', self.download_http)
|
||||
self.register_scheme('https', self.download_http)
|
||||
self.register_scheme("http", self.download_http)
|
||||
self.register_scheme("https", self.download_http)
|
||||
|
||||
def _file_download(
|
||||
self, attachment_s: str, dest_path: str, extra_s: str, profile: str
|
||||
self, attachment_s: str, dest_path: str, extra_s: str, profile: str
|
||||
) -> defer.Deferred:
|
||||
d = defer.ensureDeferred(self.file_download(
|
||||
self.host.get_client(profile),
|
||||
data_format.deserialise(attachment_s),
|
||||
Path(dest_path),
|
||||
data_format.deserialise(extra_s)
|
||||
))
|
||||
d = defer.ensureDeferred(
|
||||
self.file_download(
|
||||
self.host.get_client(profile),
|
||||
data_format.deserialise(attachment_s),
|
||||
Path(dest_path),
|
||||
data_format.deserialise(extra_s),
|
||||
)
|
||||
)
|
||||
d.addCallback(lambda ret: data_format.serialise(ret))
|
||||
return d
|
||||
|
||||
|
|
@ -92,7 +94,7 @@ class DownloadPlugin(object):
|
|||
client: SatXMPPEntity,
|
||||
attachment: Dict[str, Any],
|
||||
dest_path: Path,
|
||||
extra: Optional[Dict[str, Any]] = None
|
||||
extra: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Download a file using best available method
|
||||
|
||||
|
|
@ -103,8 +105,10 @@ class DownloadPlugin(object):
|
|||
try:
|
||||
progress_id, __ = await self.download(client, attachment, dest_path, extra)
|
||||
except Exception as e:
|
||||
if (isinstance(e, jabber_error.StanzaError)
|
||||
and e.condition == 'not-acceptable'):
|
||||
if (
|
||||
isinstance(e, jabber_error.StanzaError)
|
||||
and e.condition == "not-acceptable"
|
||||
):
|
||||
reason = e.text
|
||||
else:
|
||||
reason = str(e)
|
||||
|
|
@ -119,14 +123,16 @@ class DownloadPlugin(object):
|
|||
return {"progress": progress_id}
|
||||
|
||||
def _file_download_complete(
|
||||
self, attachment_s: str, dest_path: str, extra_s: str, profile: str
|
||||
self, attachment_s: str, dest_path: str, extra_s: str, profile: str
|
||||
) -> defer.Deferred:
|
||||
d = defer.ensureDeferred(self.file_download_complete(
|
||||
self.host.get_client(profile),
|
||||
data_format.deserialise(attachment_s),
|
||||
Path(dest_path),
|
||||
data_format.deserialise(extra_s)
|
||||
))
|
||||
d = defer.ensureDeferred(
|
||||
self.file_download_complete(
|
||||
self.host.get_client(profile),
|
||||
data_format.deserialise(attachment_s),
|
||||
Path(dest_path),
|
||||
data_format.deserialise(extra_s),
|
||||
)
|
||||
)
|
||||
d.addCallback(lambda path: str(path))
|
||||
return d
|
||||
|
||||
|
|
@ -135,7 +141,7 @@ class DownloadPlugin(object):
|
|||
client: SatXMPPEntity,
|
||||
attachment: Dict[str, Any],
|
||||
dest_path: Path,
|
||||
extra: Optional[Dict[str, Any]] = None
|
||||
extra: Optional[Dict[str, Any]] = None,
|
||||
) -> str:
|
||||
"""Helper method to fully download a file and return its path
|
||||
|
||||
|
|
@ -152,11 +158,11 @@ class DownloadPlugin(object):
|
|||
client: SatXMPPEntity,
|
||||
uri: str,
|
||||
dest_path: Union[Path, str],
|
||||
extra: Optional[Dict[str, Any]] = None
|
||||
extra: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[str, defer.Deferred]:
|
||||
if extra is None:
|
||||
extra = {}
|
||||
uri_parsed = urlparse(uri, 'http')
|
||||
uri_parsed = urlparse(uri, "http")
|
||||
if dest_path:
|
||||
dest_path = Path(dest_path)
|
||||
cache_uid = None
|
||||
|
|
@ -164,31 +170,32 @@ class DownloadPlugin(object):
|
|||
filename = Path(unquote(uri_parsed.path)).name.strip() or C.FILE_DEFAULT_NAME
|
||||
# we don't use Path.suffixes because we don't want to have more than 2
|
||||
# suffixes, but we still want to handle suffixes like "tar.gz".
|
||||
stem, *suffixes = filename.rsplit('.', 2)
|
||||
stem, *suffixes = filename.rsplit(".", 2)
|
||||
# we hash the URL to have an unique identifier, and avoid double download
|
||||
url_hash = hashlib.sha256(uri_parsed.geturl().encode()).hexdigest()
|
||||
cache_uid = f"{stem}_{url_hash}"
|
||||
cache_data = client.cache.get_metadata(cache_uid)
|
||||
if cache_data is not None:
|
||||
# file is already in cache, we return it
|
||||
download_d = defer.succeed(cache_data['path'])
|
||||
return '', download_d
|
||||
download_d = defer.succeed(cache_data["path"])
|
||||
return "", download_d
|
||||
else:
|
||||
# the file is not in cache
|
||||
unique_name = '.'.join([cache_uid] + suffixes)
|
||||
unique_name = ".".join([cache_uid] + suffixes)
|
||||
with client.cache.cache_data(
|
||||
"DOWNLOAD", cache_uid, filename=unique_name) as f:
|
||||
"DOWNLOAD", cache_uid, filename=unique_name
|
||||
) as f:
|
||||
# we close the file and only use its name, the file will be opened
|
||||
# by the registered callback
|
||||
dest_path = Path(f.name)
|
||||
|
||||
# should we check certificates?
|
||||
check_certificate = self.host.memory.param_get_a(
|
||||
"check_certificate", "Connection", profile_key=client.profile)
|
||||
"check_certificate", "Connection", profile_key=client.profile
|
||||
)
|
||||
if not check_certificate:
|
||||
extra['ignore_tls_errors'] = True
|
||||
log.warning(
|
||||
_("certificate check disabled for download, this is dangerous!"))
|
||||
extra["ignore_tls_errors"] = True
|
||||
log.warning(_("certificate check disabled for download, this is dangerous!"))
|
||||
|
||||
try:
|
||||
callback = self._scheme_callbacks[uri_parsed.scheme]
|
||||
|
|
@ -197,11 +204,12 @@ class DownloadPlugin(object):
|
|||
else:
|
||||
try:
|
||||
progress_id, download_d = await callback(
|
||||
client, uri_parsed, dest_path, extra)
|
||||
client, uri_parsed, dest_path, extra
|
||||
)
|
||||
except Exception as e:
|
||||
log.warning(_(
|
||||
"Can't download URI {uri}: {reason}").format(
|
||||
uri=uri, reason=e))
|
||||
log.warning(
|
||||
_("Can't download URI {uri}: {reason}").format(uri=uri, reason=e)
|
||||
)
|
||||
if cache_uid is not None:
|
||||
client.cache.remove_from_cache(cache_uid)
|
||||
elif dest_path.exists():
|
||||
|
|
@ -210,13 +218,12 @@ class DownloadPlugin(object):
|
|||
download_d.addCallback(lambda __: dest_path)
|
||||
return progress_id, download_d
|
||||
|
||||
|
||||
async def download(
|
||||
self,
|
||||
client: SatXMPPEntity,
|
||||
attachment: Dict[str, Any],
|
||||
dest_path: Union[Path, str],
|
||||
extra: Optional[Dict[str, Any]] = None
|
||||
extra: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[str, defer.Deferred]:
|
||||
"""Download a file from URI using suitable method
|
||||
|
||||
|
|
@ -247,9 +254,7 @@ class DownloadPlugin(object):
|
|||
try:
|
||||
cb = self._download_callbacks[source_type]
|
||||
except KeyError:
|
||||
log.warning(
|
||||
f"no source handler registered for {source_type!r}"
|
||||
)
|
||||
log.warning(f"no source handler registered for {source_type!r}")
|
||||
else:
|
||||
try:
|
||||
return await cb(client, attachment, source, dest_path, extra)
|
||||
|
|
@ -271,11 +276,14 @@ class DownloadPlugin(object):
|
|||
source_type: str,
|
||||
callback: Callable[
|
||||
[
|
||||
SatXMPPEntity, Dict[str, Any], Dict[str, Any], Union[str, Path],
|
||||
Dict[str, Any]
|
||||
SatXMPPEntity,
|
||||
Dict[str, Any],
|
||||
Dict[str, Any],
|
||||
Union[str, Path],
|
||||
Dict[str, Any],
|
||||
],
|
||||
Tuple[str, defer.Deferred]
|
||||
]
|
||||
Tuple[str, defer.Deferred],
|
||||
],
|
||||
) -> None:
|
||||
"""Register a handler to manage a type of attachment source
|
||||
|
||||
|
|
@ -334,17 +342,15 @@ class DownloadPlugin(object):
|
|||
async def download_http(self, client, uri_parsed, dest_path, options):
|
||||
url = uri_parsed.geturl()
|
||||
|
||||
if options.get('ignore_tls_errors', False):
|
||||
log.warning(
|
||||
"TLS certificate check disabled, this is highly insecure"
|
||||
)
|
||||
if options.get("ignore_tls_errors", False):
|
||||
log.warning("TLS certificate check disabled, this is highly insecure")
|
||||
treq_client = treq_client_no_ssl
|
||||
else:
|
||||
treq_client = treq
|
||||
|
||||
head_data = await treq_client.head(url)
|
||||
try:
|
||||
content_length = int(head_data.headers.getRawHeaders('content-length')[0])
|
||||
content_length = int(head_data.headers.getRawHeaders("content-length")[0])
|
||||
except (KeyError, TypeError, IndexError):
|
||||
content_length = None
|
||||
log.debug(f"No content lenght found at {url}")
|
||||
|
|
@ -353,7 +359,7 @@ class DownloadPlugin(object):
|
|||
client,
|
||||
dest_path,
|
||||
mode="wb",
|
||||
size = content_length,
|
||||
size=content_length,
|
||||
)
|
||||
|
||||
progress_id = file_obj.uid
|
||||
|
|
|
|||
Loading…
Reference in a new issue