From 92cb52dd048031afc5770a35b67c34e51eaeb955 Mon Sep 17 00:00:00 2001 From: Goffi Date: Thu, 9 Jul 2026 13:07:01 +0200 Subject: [PATCH] plugin XEP-0363, upload: compute hash on file upload: - Hash (SHA-256) is now automacally computed when a file is uploaded via XEP-0363, and used if no hash has been specified by frontend already. - Use named tuples (`UploadTracking` and `UploadResult`) to handle callback values. - upload: fix a bug (missing variable) in `unregister` (which is not used yet anyway). - usual docstring/type hints improvements --- .../backend/plugins/plugin_blog_import.py | 5 +- .../backend/plugins/plugin_misc_attach.py | 23 +-- .../backend/plugins/plugin_misc_upload.py | 134 ++++++++++++------ libervia/backend/plugins/plugin_sec_aesgcm.py | 3 +- libervia/backend/plugins/plugin_xep_0363.py | 81 ++++++++--- libervia/backend/plugins/plugin_xep_0448.py | 3 +- 6 files changed, 174 insertions(+), 75 deletions(-) diff --git a/libervia/backend/plugins/plugin_blog_import.py b/libervia/backend/plugins/plugin_blog_import.py index b3e89820..7749f89f 100644 --- a/libervia/backend/plugins/plugin_blog_import.py +++ b/libervia/backend/plugins/plugin_blog_import.py @@ -305,10 +305,11 @@ class BlogImportPlugin(object): filename = filename.replace( "%", "_" ) # FIXME: tmp workaround for a bug in prosody http upload - __, download_d = yield self._u.upload( + result = yield self._u.upload( client, tmp_file, filename, extra=upload_options ) - download_url = yield download_d + upload_result = yield result.download_d + download_url = upload_result.url except Exception as e: log.warning( "can't download image at {url}: {reason}".format(url=url, reason=e) diff --git a/libervia/backend/plugins/plugin_misc_attach.py b/libervia/backend/plugins/plugin_misc_attach.py index f49c860a..13e5771f 100644 --- a/libervia/backend/plugins/plugin_misc_attach.py +++ b/libervia/backend/plugins/plugin_misc_attach.py @@ -21,7 +21,9 @@ import mimetypes from pathlib import Path import shutil import tempfile -from typing import Callable, Optional +from typing import Callable, Optional, cast + +from libervia.backend.plugins.plugin_misc_upload import UploadPlugin, UploadResult from twisted.internet import defer @@ -30,6 +32,7 @@ from libervia.backend.core.constants import Const as C from libervia.backend.core.core_types import SatXMPPEntity from libervia.backend.core.i18n import _ from libervia.backend.core.log import getLogger +from libervia.backend.plugins.plugin_misc_upload import UploadPlugin, UploadResult from libervia.backend.tools import utils from libervia.backend.tools import image @@ -56,7 +59,7 @@ class AttachPlugin: def __init__(self, host): log.info(_("plugin Attach initialization")) self.host = host - self._u = host.plugins["UPLOAD"] + self._u = cast(UploadPlugin, host.plugins["UPLOAD"]) host.trigger.add("sendMessage", self._send_message_trigger) host.trigger.add("sendMessageComponent", self._send_message_trigger) self._attachments_handlers = {"clear": [], "encrypted": []} @@ -220,6 +223,7 @@ class AttachPlugin: attachment["size"] = path.stat().st_size extra = {"attachment": attachment} + extra["hash_algo"] = "sha-256" progress_id = attachment.pop("progress_id", None) if progress_id: extra["progress_id"] = progress_id @@ -232,27 +236,30 @@ class AttachPlugin: _("certificate check disabled for upload, this is dangerous!") ) - __, upload_d = await upload_cb( + upload_result = await upload_cb( client=client, filepath=path, filename=name, extra=extra, ) - uploads_d.append(upload_d) + uploads_d.append(upload_result.download_d) for attachment in to_delete: attachments.remove(attachment) upload_results = await defer.DeferredList(uploads_d) - for idx, (success, ret) in enumerate(upload_results): + for idx, (success, upload_result) in enumerate(upload_results): attachment = attachments[idx] if not success: - # ret is a failure here - log.warning(f"error while uploading {attachment}: {ret}") + # upload_result is a failure here + log.warning(f"error while uploading {attachment}: {upload_result}") continue - attachment["url"] = ret + attachment["url"] = upload_result.url + if upload_result.hash and "hash" not in attachment: + attachment["hash"] = upload_result.hash + attachment["hash_algo"] = upload_result.hash_algo return data diff --git a/libervia/backend/plugins/plugin_misc_upload.py b/libervia/backend/plugins/plugin_misc_upload.py index f254b9eb..6936e503 100644 --- a/libervia/backend/plugins/plugin_misc_upload.py +++ b/libervia/backend/plugins/plugin_misc_upload.py @@ -19,7 +19,7 @@ import os import os.path from pathlib import Path -from typing import Optional, Tuple, Union +from typing import Callable, NamedTuple, TYPE_CHECKING from twisted.internet import defer from twisted.words.protocols.jabber import jid @@ -33,6 +33,25 @@ from libervia.backend.core.log import getLogger from libervia.backend.tools import xml_tools from libervia.backend.tools.common import data_format +if TYPE_CHECKING: + from libervia.backend.core.main import LiberviaBackend + + +class UploadTracking(NamedTuple): + """Tracking data for an ongoing file upload""" + + progress_id: str + download_d: defer.Deferred + + +class UploadResult(NamedTuple): + """Result of a completed file upload""" + + url: str + hash: str | None = None + hash_algo: str | None = None + + log = getLogger(__name__) @@ -54,7 +73,7 @@ UPLOADING_TITLE = D_("File upload") class UploadPlugin(object): # TODO: plugin unload - def __init__(self, host): + def __init__(self, host: "LiberviaBackend") -> None: log.info(_("plugin Upload initialization")) self.host = host host.bridge.add_method( @@ -68,27 +87,49 @@ class UploadPlugin(object): self._upload_callbacks = [] def _file_upload( - self, filepath, filename, upload_jid_s="", options="", profile=C.PROF_KEY_NONE - ): + self, + filepath: str, + filename: str, + upload_jid_s: str, + options_s: str, + profile: str = C.PROF_KEY_NONE, + ) -> defer.Deferred: + """Bridge method to upload a file. + + @param filepath: absolute path to the file + @param filename: name to use for the upload + @param upload_jid_s: upload capable entity jid as string + @param options_s: serialised upload options + @param profile: profile key + @return: Deferred which fires with progress id or error + """ client = self.host.get_client(profile) upload_jid = jid.JID(upload_jid_s) if upload_jid_s else None - options = data_format.deserialise(options) + options: dict = data_format.deserialise(options_s) return defer.ensureDeferred( self.file_upload(client, filepath, filename or None, upload_jid, options) ) - async def file_upload(self, client, filepath, filename, upload_jid, options): - """Send a file using best available method + async def file_upload( + self, + client: SatXMPPEntity, + filepath: Path | str, + filename: str | None, + upload_jid: jid.JID | None, + options: dict, + ) -> dict[str, str]: + """Send a file using best available method. - parameters are the same as for [upload] - @return (dict): action dictionary, with progress id in case of success, else xmlui - message + @param client: XMPP client + @param filepath: absolute path to the file + @param filename: name to use for the upload + @param upload_jid: upload capable entity jid + @param options: upload options + @return: action dictionary with progress id in case of success, else xmlui message """ try: - progress_id, __ = await self.upload( - client, filepath, filename, upload_jid, options - ) + result = await self.upload(client, filepath, filename, upload_jid, options) except Exception as e: if ( isinstance(e, jabber_error.StanzaError) @@ -105,16 +146,16 @@ class UploadPlugin(object): ).toXml() } else: - return {"progress": progress_id} + return {"progress": result.progress_id} async def upload( self, client: SatXMPPEntity, - filepath: Union[Path, str], - filename: Optional[str] = None, - upload_jid: Optional[jid.JID] = None, - extra: Optional[dict] = None, - ) -> Tuple[str, defer.Deferred]: + filepath: Path | str, + filename: str | None = None, + upload_jid: jid.JID | None = None, + extra: dict | None = None, + ) -> UploadTracking: """Send a file using best available method @param filepath: absolute path to the file @@ -127,9 +168,8 @@ class UploadPlugin(object): used only if HTTPS transport is needed - progress_id(str): id to use for progression if not specified, one will be generated - @param profile: %(doc_profile)s - @return: progress_id and a Deferred which fire download URL when upload is - finished + @return: UploadTracking with progress_id and deferred which fires download URL + when upload is finished """ if extra is None: extra = {} @@ -145,27 +185,36 @@ class UploadPlugin(object): log.info( "{name} method will be used to upload the file".format(name=method_name) ) - progress_id, download_d = await upload_cb( - client, filepath, filename, upload_jid, extra + result = await upload_cb(client, filepath, filename, upload_jid, extra) + return UploadTracking( + progress_id=result.progress_id, + download_d=result.download_d, ) - return progress_id, download_d raise exceptions.NotFound("Can't find any method to upload a file") - def register(self, method_name, available_cb, upload_cb, priority=0): - """Register a fileUploading method + def register( + self, + method_name: str, + available_cb: Callable[[SatXMPPEntity, jid.JID | None], defer.Deferred[jid.JID]], + upload_cb: Callable[ + [SatXMPPEntity, Path | str, str | None, jid.JID, dict], + UploadTracking, + ], + priority: int = 0, + ) -> None: + """Register a file uploading method. - @param method_name(unicode): short name for the method, must be unique - @param available_cb(callable): method to call to check if this method is usable - the callback must take two arguments: upload_jid (can be None) and profile - the callback must return the first entity found (being upload_jid or one of its - components) - exceptions.NotFound must be raised if no entity has been found - @param upload_cb(callable): method to upload a file - must have the same signature as [file_upload] - must return a tuple with progress_id and a Deferred which fire download URL - when upload is finished - @param priority(int): pririoty of this method, the higher available will be used + @param method_name: short name for the method, must be unique + @param available_cb: method to call to check if this method is usable + must take two arguments: client and upload_jid (can be None) + must return the first entity found (being upload_jid or one of its components) + exceptions.NotFound must be raised if no entity has been found + @param upload_cb: method to upload a file + must have the same signature as [upload] + must return an UploadTracking with progress_id and deferred which fires + download URL when upload is finished + @param priority: priority of this method, the higher available will be used """ assert method_name for data in self._upload_callbacks: @@ -176,9 +225,14 @@ class UploadPlugin(object): self._upload_callbacks.append((method_name, available_cb, upload_cb, priority)) self._upload_callbacks.sort(key=lambda data: data[3], reverse=True) - def unregister(self, method_name): + def unregister(self, method_name: str) -> None: + """Unregister a previously registered upload method. + + @param method_name: name of the method to unregister + @raise NotFound: if the method is not registered + """ for idx, data in enumerate(self._upload_callbacks): if data[0] == method_name: - del [idx] + del self._upload_callbacks[idx] return raise exceptions.NotFound("The name to unregister doesn't exist") diff --git a/libervia/backend/plugins/plugin_sec_aesgcm.py b/libervia/backend/plugins/plugin_sec_aesgcm.py index 6b2f9a43..e78ba703 100644 --- a/libervia/backend/plugins/plugin_sec_aesgcm.py +++ b/libervia/backend/plugins/plugin_sec_aesgcm.py @@ -143,9 +143,10 @@ class AESGCM(object): async def _upload_cb(self, client, filepath, filename, extra): extra["encryption"] = C.ENC_AES_GCM - return await self._http_upload.file_http_upload( + result = await self._http_upload.file_http_upload( client=client, filepath=filepath, filename=filename, extra=extra ) + return result.progress_id, result.download_d async def attach(self, client, data): # XXX: the attachment removal/resend code below is due to the one file per diff --git a/libervia/backend/plugins/plugin_xep_0363.py b/libervia/backend/plugins/plugin_xep_0363.py index f4e1cf61..5b274fb7 100644 --- a/libervia/backend/plugins/plugin_xep_0363.py +++ b/libervia/backend/plugins/plugin_xep_0363.py @@ -20,11 +20,11 @@ from dataclasses import dataclass import mimetypes import os.path from pathlib import Path -from typing import Callable, NamedTuple, Optional, Tuple +from typing import Callable, NamedTuple, Optional, cast from urllib import parse -from twisted.internet import reactor -from twisted.internet import defer +from twisted.internet import defer, reactor +from twisted.python.failure import Failure from twisted.web import client as http_client from twisted.web import http_headers from twisted.words.protocols.jabber import error, jid, xmlstream @@ -38,7 +38,11 @@ from libervia.backend.core.core_types import SatXMPPEntity from libervia.backend.core.i18n import _ from libervia.backend.core.log import getLogger from libervia.backend.core.xmpp import SatXMPPComponent +from libervia.backend.plugins.plugin_misc_file import FilePlugin +from libervia.backend.plugins.plugin_misc_upload import UploadResult, UploadTracking +from libervia.backend.plugins.plugin_xep_0300 import Hasher, XEP_0300 from libervia.backend.tools import utils, web as sat_web +from libervia.backend.tools.stream import SatFile log = getLogger(__name__) @@ -86,6 +90,8 @@ class XEP_0363: def __init__(self, host): log.info(_("plugin HTTP File Upload initialization")) self.host = host + self._file = cast(FilePlugin, self.host.plugins["FILE"]) + self._h = cast(XEP_0300, self.host.plugins["XEP-0300"]) host.bridge.add_method( "file_http_upload", ".plugin", @@ -179,10 +185,10 @@ class XEP_0363: self, client: SatXMPPEntity, filepath: Path, - filename: Optional[str] = None, - upload_jid: Optional[jid.JID] = None, - extra: Optional[dict] = None, - ) -> Tuple[str, defer.Deferred]: + filename: str | None = None, + upload_jid: jid.JID | None = None, + extra: dict | None = None, + ) -> UploadTracking: """Upload a file through HTTP @param filepath: absolute path of the file @@ -193,7 +199,8 @@ class XEP_0363: @param extra: options where key can be: - ignore_tls_errors(bool): if True, SSL certificate will not be checked - attachment(dict): file attachment data - @param profile: %(doc_profile)s + - hash_algo(str): algorithm name to compute hash during upload. + Must be one of the values supported by ``XEP-0300`` (e.g. "sha-256"). @return: progress id and Deferred which fire download URL """ if extra is None: @@ -226,12 +233,20 @@ class XEP_0363: raise e else: log.debug(f"Got upload slot: {slot}") - sat_file = self.host.plugins["FILE"].File( + hash_algo = extra.get("hash_algo") + if hash_algo: + hash_hasher = self._h.get_hasher(hash_algo) + data_cb = hash_hasher.update + else: + data_cb = None + hash_hasher = None + sat_file = self._file.File( self.host, client, filepath, uid=extra.get("progress_id"), size=file_metadata["size"], + data_cb=data_cb, auto_end_signals=False, ) progress_id = sat_file.uid @@ -269,29 +284,49 @@ class XEP_0363: download_d.addCallbacks( self._upload_cb, self._upload_eb, - (sat_file, slot), - None, + (sat_file, slot, hash_hasher, hash_algo, extra), + {}, (sat_file,), ) - return progress_id, download_d + return UploadTracking(progress_id, download_d) - def _upload_cb(self, __, sat_file, slot): - """Called once file is successfully uploaded + def _upload_cb( + self, + __, + sat_file: SatFile, + slot: Slot, + hash_hasher: Hasher | None, + hash_algo: str | None, + extra: dict, + ) -> UploadResult: + """Called once file is successfully uploaded. - @param sat_file(SatFile): file used for the upload - should be closed, but it is needed to send the progress_finished signal - @param slot(Slot): put/get urls + @param sat_file: file used for the upload + @param slot: put/get URLs + @param hash_hasher: hasher instance used during upload (may be None) + @param hash_algo: algorithm used for hashing + @param extra: extra dict, updated with hash on success + @return: upload result with URL and hash """ log.info(f"HTTP upload finished ({slot.get})") - sat_file.progress_finished({"url": slot.get}) - return slot.get + progress_data: dict[str, str] = {"url": slot.get} + if hash_hasher: + assert hash_algo is not None + progress_data["hash"] = hash_hasher.hexdigest() + progress_data["hash_algo"] = hash_algo + sat_file.progress_finished(progress_data) + return UploadResult( + url=slot.get, + hash=progress_data.get("hash"), + hash_algo=progress_data.get("hash_algo"), + ) - def _upload_eb(self, failure_, sat_file): - """Called on unsuccessful upload + def _upload_eb(self, failure_: Failure, sat_file: SatFile) -> None: + """Called on unsuccessful upload. - @param sat_file(SatFile): file used for the upload - should be closed, be is needed to send the progress_error signal + @param failure_: the failure that caused the upload to fail + @param sat_file: file used for the upload """ try: wrapped_fail = failure_.value.reasons[0] diff --git a/libervia/backend/plugins/plugin_xep_0448.py b/libervia/backend/plugins/plugin_xep_0448.py index 4c1cf5cc..0867e8de 100644 --- a/libervia/backend/plugins/plugin_xep_0448.py +++ b/libervia/backend/plugins/plugin_xep_0448.py @@ -290,9 +290,10 @@ class XEP_0448: "key": secrets.token_bytes(32), } attachment["filename"] = filename - return await self._http_upload.file_http_upload( + result = await self._http_upload.file_http_upload( client=client, filepath=filepath, filename="encrypted", extra=extra ) + return result.progress_id, result.download_d async def attach(self, client, data): # XXX: for now, XEP-0447/XEP-0448 only allow to send one file per , thus