erebion/libervia-backend
Watch
1
0
Fork
You've already forked libervia-backend
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

plugin XEP-0363, upload: compute hash on file upload:

Browse source 
- Hash (SHA-256) is now automacally computed when a file is uploaded via XEP-0363, and
  used if no hash has been specified by frontend already.
- Use named tuples (`UploadTracking` and `UploadResult`) to handle callback values.
- upload: fix a bug (missing variable) in `unregister` (which is not used yet anyway).
- usual docstring/type hints improvements
This commit is contained in:
Goffi 2026-07-09 13:07:01 +02:00
commit 92cb52dd04
6 changed files with 177 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

5
libervia/backend/plugins/plugin_blog_import.py
View file

@ -305,10 +305,11 @@ class BlogImportPlugin(object):
filename = filename.replace(
"%", "_"
) # FIXME: tmp workaround for a bug in prosody http upload
__, download_d = yield self._u.upload(
result = yield self._u.upload(
client, tmp_file, filename, extra=upload_options
)
download_url = yield download_d
upload_result = yield result.download_d
download_url = upload_result.url
except Exception as e:
log.warning(
"can't download image at {url}: {reason}".format(url=url, reason=e)

23
libervia/backend/plugins/plugin_misc_attach.py
View file

@ -21,7 +21,9 @@ import mimetypes
from pathlib import Path
import shutil
import tempfile
from typing import Callable, Optional
from typing import Callable, Optional, cast
from libervia.backend.plugins.plugin_misc_upload import UploadPlugin, UploadResult
from twisted.internet import defer
@ -30,6 +32,7 @@ from libervia.backend.core.constants import Const as C
from libervia.backend.core.core_types import SatXMPPEntity
from libervia.backend.core.i18n import _
from libervia.backend.core.log import getLogger
from libervia.backend.plugins.plugin_misc_upload import UploadPlugin, UploadResult
from libervia.backend.tools import utils
from libervia.backend.tools import image
@ -56,7 +59,7 @@ class AttachPlugin:
def __init__(self, host):
log.info(_("plugin Attach initialization"))
self.host = host
self._u = host.plugins["UPLOAD"]
self._u = cast(UploadPlugin, host.plugins["UPLOAD"])
host.trigger.add("sendMessage", self._send_message_trigger)
host.trigger.add("sendMessageComponent", self._send_message_trigger)
self._attachments_handlers = {"clear": [], "encrypted": []}
@ -220,6 +223,7 @@ class AttachPlugin:
attachment["size"] = path.stat().st_size
extra = {"attachment": attachment}
extra["hash_algo"] = "sha-256"
progress_id = attachment.pop("progress_id", None)
if progress_id:
extra["progress_id"] = progress_id
@ -232,27 +236,30 @@ class AttachPlugin:
_("certificate check disabled for upload, this is dangerous!")
)
__, upload_d = await upload_cb(
upload_result = await upload_cb(
client=client,
filepath=path,
filename=name,
extra=extra,
)
uploads_d.append(upload_d)
uploads_d.append(upload_result.download_d)
for attachment in to_delete:
attachments.remove(attachment)
upload_results = await defer.DeferredList(uploads_d)
for idx, (success, ret) in enumerate(upload_results):
for idx, (success, upload_result) in enumerate(upload_results):
attachment = attachments[idx]
if not success:
# ret is a failure here
log.warning(f"error while uploading {attachment}: {ret}")
# upload_result is a failure here
log.warning(f"error while uploading {attachment}: {upload_result}")
continue
attachment["url"] = ret
attachment["url"] = upload_result.url
if upload_result.hash and "hash" not in attachment:
attachment["hash"] = upload_result.hash
attachment["hash_algo"] = upload_result.hash_algo
return data

134
libervia/backend/plugins/plugin_misc_upload.py
View file

@ -19,7 +19,7 @@
import os
import os.path
from pathlib import Path
from typing import Optional, Tuple, Union
from typing import Callable, NamedTuple, TYPE_CHECKING
from twisted.internet import defer
from twisted.words.protocols.jabber import jid
@ -33,6 +33,25 @@ from libervia.backend.core.log import getLogger
from libervia.backend.tools import xml_tools
from libervia.backend.tools.common import data_format
if TYPE_CHECKING:
from libervia.backend.core.main import LiberviaBackend
class UploadTracking(NamedTuple):
"""Tracking data for an ongoing file upload"""
progress_id: str
download_d: defer.Deferred
class UploadResult(NamedTuple):
"""Result of a completed file upload"""
url: str
hash: str | None = None
hash_algo: str | None = None
log = getLogger(__name__)
@ -54,7 +73,7 @@ UPLOADING_TITLE = D_("File upload")
class UploadPlugin(object):
# TODO: plugin unload
def __init__(self, host):
def __init__(self, host: "LiberviaBackend") -> None:
log.info(_("plugin Upload initialization"))
self.host = host
host.bridge.add_method(
@ -68,27 +87,49 @@ class UploadPlugin(object):
self._upload_callbacks = []
def _file_upload(
self, filepath, filename, upload_jid_s="", options="", profile=C.PROF_KEY_NONE
):
self,
filepath: str,
filename: str,
upload_jid_s: str,
options_s: str,
profile: str = C.PROF_KEY_NONE,
) -> defer.Deferred:
"""Bridge method to upload a file.
@param filepath: absolute path to the file
@param filename: name to use for the upload
@param upload_jid_s: upload capable entity jid as string
@param options_s: serialised upload options
@param profile: profile key
@return: Deferred which fires with progress id or error
"""
client = self.host.get_client(profile)
upload_jid = jid.JID(upload_jid_s) if upload_jid_s else None
options = data_format.deserialise(options)
options: dict = data_format.deserialise(options_s)
return defer.ensureDeferred(
self.file_upload(client, filepath, filename or None, upload_jid, options)
)
async def file_upload(self, client, filepath, filename, upload_jid, options):
"""Send a file using best available method
async def file_upload(
self,
client: SatXMPPEntity,
filepath: Path | str,
filename: str | None,
upload_jid: jid.JID | None,
options: dict,
) -> dict[str, str]:
"""Send a file using best available method.
parameters are the same as for [upload]
@return (dict): action dictionary, with progress id in case of success, else xmlui
message
@param client: XMPP client
@param filepath: absolute path to the file
@param filename: name to use for the upload
@param upload_jid: upload capable entity jid
@param options: upload options
@return: action dictionary with progress id in case of success, else xmlui message
"""
try:
progress_id, __ = await self.upload(
client, filepath, filename, upload_jid, options
)
result = await self.upload(client, filepath, filename, upload_jid, options)
except Exception as e:
if (
isinstance(e, jabber_error.StanzaError)
@ -105,16 +146,16 @@ class UploadPlugin(object):
).toXml()
}
else:
return {"progress": progress_id}
return {"progress": result.progress_id}
async def upload(
self,
client: SatXMPPEntity,
filepath: Union[Path, str],
filename: Optional[str] = None,
upload_jid: Optional[jid.JID] = None,
extra: Optional[dict] = None,
) -> Tuple[str, defer.Deferred]:
filepath: Path | str,
filename: str | None = None,
upload_jid: jid.JID | None = None,
extra: dict | None = None,
) -> UploadTracking:
"""Send a file using best available method
@param filepath: absolute path to the file
@ -127,9 +168,8 @@ class UploadPlugin(object):
used only if HTTPS transport is needed
- progress_id(str): id to use for progression
if not specified, one will be generated
@param profile: %(doc_profile)s
@return: progress_id and a Deferred which fire download URL when upload is
finished
@return: UploadTracking with progress_id and deferred which fires download URL
when upload is finished
"""
if extra is None:
extra = {}
@ -145,27 +185,36 @@ class UploadPlugin(object):
log.info(
"{name} method will be used to upload the file".format(name=method_name)
)
progress_id, download_d = await upload_cb(
client, filepath, filename, upload_jid, extra
result = await upload_cb(client, filepath, filename, upload_jid, extra)
return UploadTracking(
progress_id=result.progress_id,
download_d=result.download_d,
)
return progress_id, download_d
raise exceptions.NotFound("Can't find any method to upload a file")
def register(self, method_name, available_cb, upload_cb, priority=0):
"""Register a fileUploading method
def register(
self,
method_name: str,
available_cb: Callable[[SatXMPPEntity, jid.JID | None], defer.Deferred[jid.JID]],
upload_cb: Callable[
[SatXMPPEntity, Path | str, str | None, jid.JID, dict],
UploadTracking,
],
priority: int = 0,
) -> None:
"""Register a file uploading method.
@param method_name(unicode): short name for the method, must be unique
@param available_cb(callable): method to call to check if this method is usable
the callback must take two arguments: upload_jid (can be None) and profile
the callback must return the first entity found (being upload_jid or one of its
components)
exceptions.NotFound must be raised if no entity has been found
@param upload_cb(callable): method to upload a file
must have the same signature as [file_upload]
must return a tuple with progress_id and a Deferred which fire download URL
when upload is finished
@param priority(int): pririoty of this method, the higher available will be used
@param method_name: short name for the method, must be unique
@param available_cb: method to call to check if this method is usable
must take two arguments: client and upload_jid (can be None)
must return the first entity found (being upload_jid or one of its components)
exceptions.NotFound must be raised if no entity has been found
@param upload_cb: method to upload a file
must have the same signature as [upload]
must return an UploadTracking with progress_id and deferred which fires
download URL when upload is finished
@param priority: priority of this method, the higher available will be used
"""
assert method_name
for data in self._upload_callbacks:
@ -176,9 +225,14 @@ class UploadPlugin(object):
self._upload_callbacks.append((method_name, available_cb, upload_cb, priority))
self._upload_callbacks.sort(key=lambda data: data[3], reverse=True)
def unregister(self, method_name):
def unregister(self, method_name: str) -> None:
"""Unregister a previously registered upload method.
@param method_name: name of the method to unregister
@raise NotFound: if the method is not registered
"""
for idx, data in enumerate(self._upload_callbacks):
if data[0] == method_name:
del [idx]
del self._upload_callbacks[idx]
return
raise exceptions.NotFound("The name to unregister doesn't exist")

3
libervia/backend/plugins/plugin_sec_aesgcm.py
View file

@ -143,9 +143,10 @@ class AESGCM(object):
async def _upload_cb(self, client, filepath, filename, extra):
extra["encryption"] = C.ENC_AES_GCM
return await self._http_upload.file_http_upload(
result = await self._http_upload.file_http_upload(
client=client, filepath=filepath, filename=filename, extra=extra
)
return result.progress_id, result.download_d
async def attach(self, client, data):
# XXX: the attachment removal/resend code below is due to the one file per

81
libervia/backend/plugins/plugin_xep_0363.py
View file

@ -20,11 +20,11 @@ from dataclasses import dataclass
import mimetypes
import os.path
from pathlib import Path
from typing import Callable, NamedTuple, Optional, Tuple
from typing import Callable, NamedTuple, Optional, cast
from urllib import parse
from twisted.internet import reactor
from twisted.internet import defer
from twisted.internet import defer, reactor
from twisted.python.failure import Failure
from twisted.web import client as http_client
from twisted.web import http_headers
from twisted.words.protocols.jabber import error, jid, xmlstream
@ -38,7 +38,11 @@ from libervia.backend.core.core_types import SatXMPPEntity
from libervia.backend.core.i18n import _
from libervia.backend.core.log import getLogger
from libervia.backend.core.xmpp import SatXMPPComponent
from libervia.backend.plugins.plugin_misc_file import FilePlugin
from libervia.backend.plugins.plugin_misc_upload import UploadResult, UploadTracking
from libervia.backend.plugins.plugin_xep_0300 import Hasher, XEP_0300
from libervia.backend.tools import utils, web as sat_web
from libervia.backend.tools.stream import SatFile
log = getLogger(__name__)
@ -86,6 +90,8 @@ class XEP_0363:
def __init__(self, host):
log.info(_("plugin HTTP File Upload initialization"))
self.host = host
self._file = cast(FilePlugin, self.host.plugins["FILE"])
self._h = cast(XEP_0300, self.host.plugins["XEP-0300"])
host.bridge.add_method(
"file_http_upload",
".plugin",
@ -179,10 +185,10 @@ class XEP_0363:
self,
client: SatXMPPEntity,
filepath: Path,
filename: Optional[str] = None,
upload_jid: Optional[jid.JID] = None,
extra: Optional[dict] = None,
) -> Tuple[str, defer.Deferred]:
filename: str | None = None,
upload_jid: jid.JID | None = None,
extra: dict | None = None,
) -> UploadTracking:
"""Upload a file through HTTP
@param filepath: absolute path of the file
@ -193,7 +199,8 @@ class XEP_0363:
@param extra: options where key can be:
- ignore_tls_errors(bool): if True, SSL certificate will not be checked
- attachment(dict): file attachment data
@param profile: %(doc_profile)s
- hash_algo(str): algorithm name to compute hash during upload.
Must be one of the values supported by ``XEP-0300`` (e.g. "sha-256").
@return: progress id and Deferred which fire download URL
"""
if extra is None:
@ -226,12 +233,20 @@ class XEP_0363:
raise e
else:
log.debug(f"Got upload slot: {slot}")
sat_file = self.host.plugins["FILE"].File(
hash_algo = extra.get("hash_algo")
if hash_algo:
hash_hasher = self._h.get_hasher(hash_algo)
data_cb = hash_hasher.update
else:
data_cb = None
hash_hasher = None
sat_file = self._file.File(
self.host,
client,
filepath,
uid=extra.get("progress_id"),
size=file_metadata["size"],
data_cb=data_cb,
auto_end_signals=False,
)
progress_id = sat_file.uid
@ -269,29 +284,49 @@ class XEP_0363:
download_d.addCallbacks(
self._upload_cb,
self._upload_eb,
(sat_file, slot),
None,
(sat_file, slot, hash_hasher, hash_algo, extra),
{},
(sat_file,),
)
return progress_id, download_d
return UploadTracking(progress_id, download_d)
def _upload_cb(self, __, sat_file, slot):
"""Called once file is successfully uploaded
def _upload_cb(
self,
__,
sat_file: SatFile,
slot: Slot,
hash_hasher: Hasher | None,
hash_algo: str | None,
extra: dict,
) -> UploadResult:
"""Called once file is successfully uploaded.
@param sat_file(SatFile): file used for the upload
should be closed, but it is needed to send the progress_finished signal
@param slot(Slot): put/get urls
@param sat_file: file used for the upload
@param slot: put/get URLs
@param hash_hasher: hasher instance used during upload (may be None)
@param hash_algo: algorithm used for hashing
@param extra: extra dict, updated with hash on success
@return: upload result with URL and hash
"""
log.info(f"HTTP upload finished ({slot.get})")
sat_file.progress_finished({"url": slot.get})
return slot.get
progress_data: dict[str, str] = {"url": slot.get}
if hash_hasher:
assert hash_algo is not None
progress_data["hash"] = hash_hasher.hexdigest()
progress_data["hash_algo"] = hash_algo
sat_file.progress_finished(progress_data)
return UploadResult(
url=slot.get,
hash=progress_data.get("hash"),
hash_algo=progress_data.get("hash_algo"),
)
def _upload_eb(self, failure_, sat_file):
"""Called on unsuccessful upload
def _upload_eb(self, failure_: Failure, sat_file: SatFile) -> None:
"""Called on unsuccessful upload.
@param sat_file(SatFile): file used for the upload
should be closed, be is needed to send the progress_error signal
@param failure_: the failure that caused the upload to fail
@param sat_file: file used for the upload
"""
try:
wrapped_fail = failure_.value.reasons[0]

3
libervia/backend/plugins/plugin_xep_0448.py
View file

@ -290,9 +290,10 @@ class XEP_0448:
"key": secrets.token_bytes(32),
}
attachment["filename"] = filename
return await self._http_upload.file_http_upload(
result = await self._http_upload.file_http_upload(
client=client, filepath=filepath, filename="encrypted", extra=extra
)
return result.progress_id, result.download_d
async def attach(self, client, data):
# XXX: for now, XEP-0447/XEP-0448 only allow to send one file per <message/>, thus