core: get rid of pickle:
Use of `pickle` to serialise data was a technical legacy that was causing trouble to store in database, to update (if a class was serialised, a change could break update), and to security (pickle can lead to code execution). This patch remove all use of Pickle in favour in JSON, notably: - for caching data, a Pydantic model is now used instead - for SQLAlchemy model, the LegacyPickle is replaced by JSON serialisation - in XEP-0373 a class `PublicKeyMetadata` was serialised. New method `from_dict` and `to_dict` method have been implemented to do serialisation. - new methods to (de)serialise data can now be specified with Identity data types. It is notably used to (de)serialise `path` of avatars. A migration script has been created to convert data (for upgrade or downgrade), with special care for XEP-0373 case. Depending of size of database, this migration script can be long to run. rel 443
This commit is contained in:
parent
672f967080
commit
a8348576ae
1 changed files with 344 additions and 192 deletions
|
|
@ -85,6 +85,8 @@ class Identity:
|
|||
# we store the metadata in database, to restore it on next connection
|
||||
# (it is stored only for roster entities)
|
||||
"store": True,
|
||||
"store_serialisation": self._avatar_ser,
|
||||
"store_deserialisation": self._avatar_deser
|
||||
},
|
||||
"nicknames": {
|
||||
"type": list,
|
||||
|
|
@ -167,10 +169,16 @@ class Identity:
|
|||
|
||||
for key, value in stored_data.items():
|
||||
entity_s, name = key.split('\n')
|
||||
if name not in self.metadata.keys():
|
||||
try:
|
||||
metadata = self.metadata[name]
|
||||
except KeyError:
|
||||
log.debug(f"removing {key} from storage: not an allowed metadata name")
|
||||
to_delete.append(key)
|
||||
continue
|
||||
if value is not None:
|
||||
deser_method = metadata.get("store_deserialisation")
|
||||
if deser_method is not None:
|
||||
value = deser_method(value)
|
||||
entity = jid.JID(entity_s)
|
||||
|
||||
if name == 'avatar':
|
||||
|
|
@ -365,6 +373,10 @@ class Identity:
|
|||
client, entity, metadata_name, data)
|
||||
|
||||
if metadata.get('store', False):
|
||||
if data is not None:
|
||||
ser_method = metadata.get("store_serialisation")
|
||||
if ser_method is not None:
|
||||
data = ser_method(data)
|
||||
key = f"{entity}\n{metadata_name}"
|
||||
await client._identity_storage.aset(key, data)
|
||||
|
||||
|
|
@ -488,6 +500,10 @@ class Identity:
|
|||
|
||||
if metadata.get('store', False):
|
||||
key = f"{entity}\n{metadata_name}"
|
||||
if data is not None:
|
||||
ser_method = metadata.get("store_serialisation")
|
||||
if ser_method is not None:
|
||||
data = ser_method(data)
|
||||
await client._identity_storage.aset(key, data)
|
||||
|
||||
def default_update_is_new_data(self, client, entity, cached_data, new_data):
|
||||
|
|
@ -633,6 +649,18 @@ class Identity:
|
|||
raise ValueError(f"missing avatar data keys: {mandatory_keys - data.keys()}")
|
||||
return data
|
||||
|
||||
def _avatar_ser(self, data: dict) -> dict:
|
||||
if data.get("path"):
|
||||
# Path instance can't be stored
|
||||
data = data.copy()
|
||||
data["path"] = str(data["path"])
|
||||
return data
|
||||
|
||||
def _avatar_deser(self, data: dict) -> dict:
|
||||
if data.get("path"):
|
||||
data["path"] = Path(data["path"])
|
||||
return data
|
||||
|
||||
async def nicknames_get_post_treatment(self, client, entity, plugin_nicknames):
|
||||
"""Prepend nicknames from core locations + set default nickname
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue