plugin download: use cache if dest_path is empty:
if dest_path is not set, the file will be downloaded in cache. A new name will be generated using a hash of the uri. If the file is already downloded in cache, it won't be downloaded again (in this case returned progress_id is an empty string).
This commit is contained in:
parent
c16dd25fc9
commit
f71b40f86f
2 changed files with 43 additions and 9 deletions
|
|
@ -346,6 +346,8 @@ class Const(object):
|
||||||
## Files ##
|
## Files ##
|
||||||
FILE_TYPE_DIRECTORY = "directory"
|
FILE_TYPE_DIRECTORY = "directory"
|
||||||
FILE_TYPE_FILE = "file"
|
FILE_TYPE_FILE = "file"
|
||||||
|
# when filename can't be found automatically, this one will be used
|
||||||
|
FILE_DEFAULT_NAME = "unnamed"
|
||||||
|
|
||||||
## Permissions management ##
|
## Permissions management ##
|
||||||
ACCESS_PERM_READ = "read"
|
ACCESS_PERM_READ = "read"
|
||||||
|
|
@ -378,10 +380,13 @@ class Const(object):
|
||||||
## Misc ##
|
## Misc ##
|
||||||
SAVEFILE_DATABASE = APP_NAME_FILE + ".db"
|
SAVEFILE_DATABASE = APP_NAME_FILE + ".db"
|
||||||
IQ_SET = '/iq[@type="set"]'
|
IQ_SET = '/iq[@type="set"]'
|
||||||
ENV_PREFIX = "SAT_" # Prefix used for environment variables
|
# Prefix used for environment variables
|
||||||
|
ENV_PREFIX = "SAT_"
|
||||||
IGNORE = "ignore"
|
IGNORE = "ignore"
|
||||||
NO_LIMIT = -1 # used in bridge when a integer value is expected
|
# used in bridge when a integer value is expected
|
||||||
DEFAULT_MAX_AGE = 1209600 # default max age of cached files, in seconds
|
NO_LIMIT = -1
|
||||||
|
# default max age of cached files, in seconds
|
||||||
|
DEFAULT_MAX_AGE = 3600 * 24 * 14
|
||||||
HASH_SHA1_EMPTY = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
HASH_SHA1_EMPTY = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
||||||
STANZA_NAMES = ("iq", "message", "presence")
|
STANZA_NAMES = ("iq", "message", "presence")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,8 @@
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse, unquote
|
||||||
|
import hashlib
|
||||||
import treq
|
import treq
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
from twisted.words.protocols.jabber import error as jabber_error
|
from twisted.words.protocols.jabber import error as jabber_error
|
||||||
|
|
@ -76,7 +77,7 @@ class DownloadPlugin(object):
|
||||||
))
|
))
|
||||||
|
|
||||||
async def fileDownload(self, client, uri, dest_path, options=None):
|
async def fileDownload(self, client, uri, dest_path, options=None):
|
||||||
"""Send a file using best available method
|
"""Download a file using best available method
|
||||||
|
|
||||||
parameters are the same as for [download]
|
parameters are the same as for [download]
|
||||||
@return (dict): action dictionary, with progress id in case of success, else xmlui
|
@return (dict): action dictionary, with progress id in case of success, else xmlui
|
||||||
|
|
@ -105,7 +106,7 @@ class DownloadPlugin(object):
|
||||||
options = data_format.deserialise(options_s)
|
options = data_format.deserialise(options_s)
|
||||||
|
|
||||||
d = defer.ensureDeferred(self.fileDownloadComplete(
|
d = defer.ensureDeferred(self.fileDownloadComplete(
|
||||||
client, uri, Path(dest_path), options
|
client, uri, dest_path, options
|
||||||
))
|
))
|
||||||
d.addCallback(lambda path: str(path))
|
d.addCallback(lambda path: str(path))
|
||||||
return d
|
return d
|
||||||
|
|
@ -115,9 +116,10 @@ class DownloadPlugin(object):
|
||||||
|
|
||||||
parameters are the same as for [download]
|
parameters are the same as for [download]
|
||||||
@return (str): path to the downloaded file
|
@return (str): path to the downloaded file
|
||||||
|
use empty string to store the file in cache
|
||||||
"""
|
"""
|
||||||
__, download_d = await self.download(client, uri, dest_path, options)
|
__, download_d = await self.download(client, uri, dest_path, options)
|
||||||
await download_d
|
dest_path = await download_d
|
||||||
return dest_path
|
return dest_path
|
||||||
|
|
||||||
async def download(self, client, uri, dest_path, options=None):
|
async def download(self, client, uri, dest_path, options=None):
|
||||||
|
|
@ -125,24 +127,51 @@ class DownloadPlugin(object):
|
||||||
|
|
||||||
@param uri(str): URI to the file to download
|
@param uri(str): URI to the file to download
|
||||||
@param dest_path(str, Path): where the file must be downloaded
|
@param dest_path(str, Path): where the file must be downloaded
|
||||||
|
if empty string, the file will be stored in local path
|
||||||
@param options(dict, None): options depending on scheme handler
|
@param options(dict, None): options depending on scheme handler
|
||||||
Some common options:
|
Some common options:
|
||||||
- ignore_tls_errors(bool): True to ignore SSL/TLS certificate verification
|
- ignore_tls_errors(bool): True to ignore SSL/TLS certificate verification
|
||||||
used only if HTTPS transport is needed
|
used only if HTTPS transport is needed
|
||||||
@return (tuple[unicode,D(unicode)]): progress_id and a Deferred which fire
|
@return (tuple[unicode,D(unicode)]): progress_id and a Deferred which fire
|
||||||
download URL when download is finished
|
download URL when download is finished
|
||||||
|
progress_id can be empty string if the file already exist and is not
|
||||||
|
downloaded again (can happen if cache is used with empty dest_path)
|
||||||
"""
|
"""
|
||||||
if options is None:
|
if options is None:
|
||||||
options = {}
|
options = {}
|
||||||
|
|
||||||
dest_path = Path(dest_path)
|
|
||||||
uri_parsed = urlparse(uri, 'http')
|
uri_parsed = urlparse(uri, 'http')
|
||||||
|
if dest_path:
|
||||||
|
dest_path = Path(dest_path)
|
||||||
|
else:
|
||||||
|
filename = Path(unquote(uri_parsed.path)).name.strip() or C.FILE_DEFAULT_NAME
|
||||||
|
# we don't use Path.suffixes because we don't want to have more than 2
|
||||||
|
# suffixes, but we still want to handle suffixes like "tar.gz".
|
||||||
|
stem, *suffixes = filename.rsplit('.', 2)
|
||||||
|
# we hash the URL to have an unique identifier, and avoid double download
|
||||||
|
url_hash = hashlib.sha256(uri_parsed.geturl().encode()).hexdigest()
|
||||||
|
uid = f"{stem}_{url_hash}"
|
||||||
|
cache_data = client.cache.getMetadata(uid)
|
||||||
|
if cache_data is not None:
|
||||||
|
# file is already in cache, we return it
|
||||||
|
download_d = defer.succeed(cache_data['path'])
|
||||||
|
return '', download_d
|
||||||
|
else:
|
||||||
|
# the file is not in cache
|
||||||
|
unique_name = '.'.join([uid] + suffixes)
|
||||||
|
with client.cache.cacheData("DOWNLOAD", uid, filename=unique_name) as f:
|
||||||
|
# we close the file and only use its name, the file will be opened
|
||||||
|
# by the registered callback
|
||||||
|
dest_path = f.name
|
||||||
try:
|
try:
|
||||||
callback = self._download_callbacks[uri_parsed.scheme]
|
callback = self._download_callbacks[uri_parsed.scheme]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise exceptions.NotFound(f"Can't find any handler for uri {uri}")
|
raise exceptions.NotFound(f"Can't find any handler for uri {uri}")
|
||||||
else:
|
else:
|
||||||
return await callback(client, uri_parsed, dest_path, options)
|
progress_id, download_d = await callback(
|
||||||
|
client, uri_parsed, dest_path, options)
|
||||||
|
download_d.addCallback(lambda __: dest_path)
|
||||||
|
return progress_id, download_d
|
||||||
|
|
||||||
def registerScheme(self, scheme, download_cb):
|
def registerScheme(self, scheme, download_cb):
|
||||||
"""Register an URI scheme handler
|
"""Register an URI scheme handler
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue