plugin download: use cache if dest_path is empty:

if dest_path is not set, the file will be downloaded in cache. A new name will be
generated using a hash of the uri. If the file is already downloded in cache, it won't be
downloaded again (in this case returned progress_id is an empty string).
This commit is contained in:
Goffi 2020-02-26 22:03:11 +01:00
commit f71b40f86f
2 changed files with 43 additions and 9 deletions

View file

@ -346,6 +346,8 @@ class Const(object):
## Files ## ## Files ##
FILE_TYPE_DIRECTORY = "directory" FILE_TYPE_DIRECTORY = "directory"
FILE_TYPE_FILE = "file" FILE_TYPE_FILE = "file"
# when filename can't be found automatically, this one will be used
FILE_DEFAULT_NAME = "unnamed"
## Permissions management ## ## Permissions management ##
ACCESS_PERM_READ = "read" ACCESS_PERM_READ = "read"
@ -378,10 +380,13 @@ class Const(object):
## Misc ## ## Misc ##
SAVEFILE_DATABASE = APP_NAME_FILE + ".db" SAVEFILE_DATABASE = APP_NAME_FILE + ".db"
IQ_SET = '/iq[@type="set"]' IQ_SET = '/iq[@type="set"]'
ENV_PREFIX = "SAT_" # Prefix used for environment variables # Prefix used for environment variables
ENV_PREFIX = "SAT_"
IGNORE = "ignore" IGNORE = "ignore"
NO_LIMIT = -1 # used in bridge when a integer value is expected # used in bridge when a integer value is expected
DEFAULT_MAX_AGE = 1209600 # default max age of cached files, in seconds NO_LIMIT = -1
# default max age of cached files, in seconds
DEFAULT_MAX_AGE = 3600 * 24 * 14
HASH_SHA1_EMPTY = "da39a3ee5e6b4b0d3255bfef95601890afd80709" HASH_SHA1_EMPTY = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
STANZA_NAMES = ("iq", "message", "presence") STANZA_NAMES = ("iq", "message", "presence")

View file

@ -17,7 +17,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse, unquote
import hashlib
import treq import treq
from twisted.internet import defer from twisted.internet import defer
from twisted.words.protocols.jabber import error as jabber_error from twisted.words.protocols.jabber import error as jabber_error
@ -76,7 +77,7 @@ class DownloadPlugin(object):
)) ))
async def fileDownload(self, client, uri, dest_path, options=None): async def fileDownload(self, client, uri, dest_path, options=None):
"""Send a file using best available method """Download a file using best available method
parameters are the same as for [download] parameters are the same as for [download]
@return (dict): action dictionary, with progress id in case of success, else xmlui @return (dict): action dictionary, with progress id in case of success, else xmlui
@ -105,7 +106,7 @@ class DownloadPlugin(object):
options = data_format.deserialise(options_s) options = data_format.deserialise(options_s)
d = defer.ensureDeferred(self.fileDownloadComplete( d = defer.ensureDeferred(self.fileDownloadComplete(
client, uri, Path(dest_path), options client, uri, dest_path, options
)) ))
d.addCallback(lambda path: str(path)) d.addCallback(lambda path: str(path))
return d return d
@ -115,9 +116,10 @@ class DownloadPlugin(object):
parameters are the same as for [download] parameters are the same as for [download]
@return (str): path to the downloaded file @return (str): path to the downloaded file
use empty string to store the file in cache
""" """
__, download_d = await self.download(client, uri, dest_path, options) __, download_d = await self.download(client, uri, dest_path, options)
await download_d dest_path = await download_d
return dest_path return dest_path
async def download(self, client, uri, dest_path, options=None): async def download(self, client, uri, dest_path, options=None):
@ -125,24 +127,51 @@ class DownloadPlugin(object):
@param uri(str): URI to the file to download @param uri(str): URI to the file to download
@param dest_path(str, Path): where the file must be downloaded @param dest_path(str, Path): where the file must be downloaded
if empty string, the file will be stored in local path
@param options(dict, None): options depending on scheme handler @param options(dict, None): options depending on scheme handler
Some common options: Some common options:
- ignore_tls_errors(bool): True to ignore SSL/TLS certificate verification - ignore_tls_errors(bool): True to ignore SSL/TLS certificate verification
used only if HTTPS transport is needed used only if HTTPS transport is needed
@return (tuple[unicode,D(unicode)]): progress_id and a Deferred which fire @return (tuple[unicode,D(unicode)]): progress_id and a Deferred which fire
download URL when download is finished download URL when download is finished
progress_id can be empty string if the file already exist and is not
downloaded again (can happen if cache is used with empty dest_path)
""" """
if options is None: if options is None:
options = {} options = {}
dest_path = Path(dest_path)
uri_parsed = urlparse(uri, 'http') uri_parsed = urlparse(uri, 'http')
if dest_path:
dest_path = Path(dest_path)
else:
filename = Path(unquote(uri_parsed.path)).name.strip() or C.FILE_DEFAULT_NAME
# we don't use Path.suffixes because we don't want to have more than 2
# suffixes, but we still want to handle suffixes like "tar.gz".
stem, *suffixes = filename.rsplit('.', 2)
# we hash the URL to have an unique identifier, and avoid double download
url_hash = hashlib.sha256(uri_parsed.geturl().encode()).hexdigest()
uid = f"{stem}_{url_hash}"
cache_data = client.cache.getMetadata(uid)
if cache_data is not None:
# file is already in cache, we return it
download_d = defer.succeed(cache_data['path'])
return '', download_d
else:
# the file is not in cache
unique_name = '.'.join([uid] + suffixes)
with client.cache.cacheData("DOWNLOAD", uid, filename=unique_name) as f:
# we close the file and only use its name, the file will be opened
# by the registered callback
dest_path = f.name
try: try:
callback = self._download_callbacks[uri_parsed.scheme] callback = self._download_callbacks[uri_parsed.scheme]
except KeyError: except KeyError:
raise exceptions.NotFound(f"Can't find any handler for uri {uri}") raise exceptions.NotFound(f"Can't find any handler for uri {uri}")
else: else:
return await callback(client, uri_parsed, dest_path, options) progress_id, download_d = await callback(
client, uri_parsed, dest_path, options)
download_d.addCallback(lambda __: dest_path)
return progress_id, download_d
def registerScheme(self, scheme, download_cb): def registerScheme(self, scheme, download_cb):
"""Register an URI scheme handler """Register an URI scheme handler