refactoring: renamed sat package to libervia.backend

This commit is contained in:
Goffi 2023-06-02 11:49:51 +02:00
commit d2685ba11b
22 changed files with 1293 additions and 1293 deletions

View file

View file

@ -0,0 +1,281 @@
#!/usr/bin/env python3
# SAT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from io import BufferedIOBase
import mimetypes
from pathlib import Path
import pickle as pickle
import time
from typing import Any, Dict, Optional
from libervia.backend.core import exceptions
from libervia.backend.core.constants import Const as C
from libervia.backend.core.i18n import _
from libervia.backend.core.log import getLogger
from libervia.backend.tools.common import regex
log = getLogger(__name__)
DEFAULT_EXT = ".raw"
class Cache(object):
"""generic file caching"""
def __init__(self, host, profile):
"""
@param profile(unicode, None): name of the profile to set the cache for
if None, the cache will be common for all profiles
"""
self.profile = profile
path_elts = [host.memory.config_get("", "local_dir"), C.CACHE_DIR]
if profile:
path_elts.extend(["profiles", regex.path_escape(profile)])
else:
path_elts.append("common")
self.cache_dir = Path(*path_elts)
self.cache_dir.mkdir(0o700, parents=True, exist_ok=True)
self.purge()
def purge(self):
# remove expired files from cache
# TODO: this should not be called only on startup, but at regular interval
# (e.g. once a day)
purged = set()
# we sort files to have metadata files first
for cache_file in sorted(self.cache_dir.iterdir()):
if cache_file in purged:
continue
try:
with cache_file.open('rb') as f:
cache_data = pickle.load(f)
except IOError:
log.warning(
_("Can't read metadata file at {path}")
.format(path=cache_file))
continue
except (pickle.UnpicklingError, EOFError):
log.debug(f"File at {cache_file} is not a metadata file")
continue
try:
eol = cache_data['eol']
filename = cache_data['filename']
except KeyError:
log.warning(
_("Invalid cache metadata at {path}")
.format(path=cache_file))
continue
filepath = self.getPath(filename)
if not filepath.exists():
log.warning(_(
"cache {cache_file!r} references an inexisting file: {filepath!r}"
).format(cache_file=str(cache_file), filepath=str(filepath)))
log.debug("purging cache with missing file")
cache_file.unlink()
elif eol < time.time():
log.debug(
"purging expired cache {filepath!r} (expired for {time}s)"
.format(filepath=str(filepath), time=int(time.time() - eol))
)
cache_file.unlink()
try:
filepath.unlink()
except FileNotFoundError:
log.warning(
_("following file is missing while purging cache: {path}")
.format(path=filepath)
)
purged.add(cache_file)
purged.add(filepath)
def getPath(self, filename: str) -> Path:
"""return cached file URL
@param filename: cached file name (cache data or actual file)
@return: path to the cached file
"""
if not filename or "/" in filename:
log.error(
"invalid char found in file name, hack attempt? name:{}".format(filename)
)
raise exceptions.DataError("Invalid char found")
return self.cache_dir / filename
def get_metadata(self, uid: str, update_eol: bool = True) -> Optional[Dict[str, Any]]:
"""Retrieve metadata for cached data
@param uid(unicode): unique identifier of file
@param update_eol(bool): True if eol must extended
if True, max_age will be added to eol (only if it is not already expired)
@return (dict, None): metadata with following keys:
see [cache_data] for data details, an additional "path" key is the full path to
cached file.
None if file is not in cache (or cache is invalid)
"""
uid = uid.strip()
if not uid:
raise exceptions.InternalError("uid must not be empty")
cache_url = self.getPath(uid)
if not cache_url.exists():
return None
try:
with cache_url.open("rb") as f:
cache_data = pickle.load(f)
except (IOError, EOFError) as e:
log.warning(f"can't read cache at {cache_url}: {e}")
return None
except pickle.UnpicklingError:
log.warning(f"invalid cache found at {cache_url}")
return None
try:
eol = cache_data["eol"]
except KeyError:
log.warning("no End Of Life found for cached file {}".format(uid))
eol = 0
if eol < time.time():
log.debug(
"removing expired cache (expired for {}s)".format(time.time() - eol)
)
return None
if update_eol:
try:
max_age = cache_data["max_age"]
except KeyError:
log.warning(f"no max_age found for cache at {cache_url}, using default")
max_age = cache_data["max_age"] = C.DEFAULT_MAX_AGE
now = int(time.time())
cache_data["last_access"] = now
cache_data["eol"] = now + max_age
with cache_url.open("wb") as f:
pickle.dump(cache_data, f, protocol=2)
cache_data["path"] = self.getPath(cache_data["filename"])
return cache_data
def get_file_path(self, uid: str) -> Path:
"""Retrieve absolute path to file
@param uid(unicode): unique identifier of file
@return (unicode, None): absolute path to cached file
None if file is not in cache (or cache is invalid)
"""
metadata = self.get_metadata(uid)
if metadata is not None:
return metadata["path"]
def remove_from_cache(self, uid, metadata=None):
"""Remove data from cache
@param uid(unicode): unique identifier cache file
"""
cache_data = self.get_metadata(uid, update_eol=False)
if cache_data is None:
log.debug(f"cache with uid {uid!r} has already expired or been removed")
return
try:
filename = cache_data['filename']
except KeyError:
log.warning(_("missing filename for cache {uid!r}") .format(uid=uid))
else:
filepath = self.getPath(filename)
try:
filepath.unlink()
except FileNotFoundError:
log.warning(
_("missing file referenced in cache {uid!r}: {filename}")
.format(uid=uid, filename=filename)
)
cache_file = self.getPath(uid)
cache_file.unlink()
log.debug(f"cache with uid {uid!r} has been removed")
def cache_data(
self,
source: str,
uid: str,
mime_type: Optional[str] = None,
max_age: Optional[int] = None,
original_filename: Optional[str] = None
) -> BufferedIOBase:
"""create cache metadata and file object to use for actual data
@param source: source of the cache (should be plugin's import_name)
@param uid: an identifier of the file which must be unique
@param mime_type: MIME type of the file to cache
it will be used notably to guess file extension
It may be autogenerated if filename is specified
@param max_age: maximum age in seconds
the cache metadata will have an "eol" (end of life)
None to use default value
0 to ignore cache (file will be re-downloaded on each access)
@param original_filename: if not None, will be used to retrieve file extension and
guess
mime type, and stored in "original_filename"
@return: file object opened in write mode
you have to close it yourself (hint: use ``with`` statement)
"""
if max_age is None:
max_age = C.DEFAULT_MAX_AGE
cache_data = {
"source": source,
# we also store max_age for updating eol
"max_age": max_age,
}
cache_url = self.getPath(uid)
if original_filename is not None:
cache_data["original_filename"] = original_filename
if mime_type is None:
# we have original_filename but not MIME type, we try to guess the later
mime_type = mimetypes.guess_type(original_filename, strict=False)[0]
if mime_type:
ext = mimetypes.guess_extension(mime_type, strict=False)
if ext is None:
log.warning(
"can't find extension for MIME type {}".format(mime_type)
)
ext = DEFAULT_EXT
elif ext == ".jpe":
ext = ".jpg"
else:
ext = DEFAULT_EXT
mime_type = None
filename = uid + ext
now = int(time.time())
cache_data.update({
"filename": filename,
"creation": now,
"eol": now + max_age,
"mime_type": mime_type,
})
file_path = self.getPath(filename)
with open(cache_url, "wb") as f:
pickle.dump(cache_data, f, protocol=2)
return file_path.open("wb")

View file

@ -0,0 +1,170 @@
#!/usr/bin/env python3
# SAT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# Copyright (C) 2013-2016 Adrien Cossa (souliane@mailoo.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from os import urandom
from base64 import b64encode, b64decode
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend
crypto_backend = default_backend()
class BlockCipher:
BLOCK_SIZE = 16
MAX_KEY_SIZE = 32
IV_SIZE = BLOCK_SIZE # initialization vector size, 16 bits
@staticmethod
def encrypt(key, text, leave_empty=True):
"""Encrypt a message.
Based on http://stackoverflow.com/a/12525165
@param key (unicode): the encryption key
@param text (unicode): the text to encrypt
@param leave_empty (bool): if True, empty text will be returned "as is"
@return (D(str)): base-64 encoded encrypted message
"""
if leave_empty and text == "":
return ""
iv = BlockCipher.get_random_key()
key = key.encode()
key = (
key[: BlockCipher.MAX_KEY_SIZE]
if len(key) >= BlockCipher.MAX_KEY_SIZE
else BlockCipher.pad(key)
)
cipher = Cipher(algorithms.AES(key), modes.CFB8(iv), backend=crypto_backend)
encryptor = cipher.encryptor()
encrypted = encryptor.update(BlockCipher.pad(text.encode())) + encryptor.finalize()
return b64encode(iv + encrypted).decode()
@staticmethod
def decrypt(key, ciphertext, leave_empty=True):
"""Decrypt a message.
Based on http://stackoverflow.com/a/12525165
@param key (unicode): the decryption key
@param ciphertext (base-64 encoded str): the text to decrypt
@param leave_empty (bool): if True, empty ciphertext will be returned "as is"
@return: Deferred: str or None if the password could not be decrypted
"""
if leave_empty and ciphertext == "":
return ""
ciphertext = b64decode(ciphertext)
iv, ciphertext = (
ciphertext[: BlockCipher.IV_SIZE],
ciphertext[BlockCipher.IV_SIZE :],
)
key = key.encode()
key = (
key[: BlockCipher.MAX_KEY_SIZE]
if len(key) >= BlockCipher.MAX_KEY_SIZE
else BlockCipher.pad(key)
)
cipher = Cipher(algorithms.AES(key), modes.CFB8(iv), backend=crypto_backend)
decryptor = cipher.decryptor()
decrypted = decryptor.update(ciphertext) + decryptor.finalize()
return BlockCipher.unpad(decrypted)
@staticmethod
def get_random_key(size=None, base64=False):
"""Return a random key suitable for block cipher encryption.
Note: a good value for the key length is to make it as long as the block size.
@param size: key length in bytes, positive or null (default: BlockCipher.IV_SIZE)
@param base64: if True, encode the result to base-64
@return: str (eventually base-64 encoded)
"""
if size is None or size < 0:
size = BlockCipher.IV_SIZE
key = urandom(size)
return b64encode(key) if base64 else key
@staticmethod
def pad(s):
"""Method from http://stackoverflow.com/a/12525165"""
bs = BlockCipher.BLOCK_SIZE
return s + (bs - len(s) % bs) * (chr(bs - len(s) % bs)).encode()
@staticmethod
def unpad(s):
"""Method from http://stackoverflow.com/a/12525165"""
s = s.decode()
return s[0 : -ord(s[-1])]
class PasswordHasher:
SALT_LEN = 16 # 128 bits
@staticmethod
def hash(password, salt=None, leave_empty=True):
"""Hash a password.
@param password (str): the password to hash
@param salt (base-64 encoded str): if not None, use the given salt instead of a random value
@param leave_empty (bool): if True, empty password will be returned "as is"
@return: Deferred: base-64 encoded str
"""
if leave_empty and password == "":
return ""
salt = (
b64decode(salt)[: PasswordHasher.SALT_LEN]
if salt
else urandom(PasswordHasher.SALT_LEN)
)
# we use PyCrypto's PBKDF2 arguments while porting to crytography, to stay
# compatible with existing installations. But this is temporary and we need
# to update them to more secure values.
kdf = PBKDF2HMAC(
# FIXME: SHA1() is not secure, it is used here for historical reasons
# and must be changed as soon as possible
algorithm=hashes.SHA1(),
length=16,
salt=salt,
iterations=1000,
backend=crypto_backend
)
key = kdf.derive(password.encode())
return b64encode(salt + key).decode()
@staticmethod
def verify(attempt, pwd_hash):
"""Verify a password attempt.
@param attempt (str): the attempt to check
@param pwd_hash (str): the hash of the password
@return: Deferred: boolean
"""
assert isinstance(attempt, str)
assert isinstance(pwd_hash, str)
leave_empty = pwd_hash == ""
attempt_hash = PasswordHasher.hash(attempt, pwd_hash, leave_empty)
assert isinstance(attempt_hash, str)
return attempt_hash == pwd_hash

499
libervia/backend/memory/disco.py Executable file
View file

@ -0,0 +1,499 @@
#!/usr/bin/env python3
# SAT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from typing import Optional
from libervia.backend.core.i18n import _
from libervia.backend.core import exceptions
from libervia.backend.core.log import getLogger
from libervia.backend.core.core_types import SatXMPPEntity
from twisted.words.protocols.jabber import jid
from twisted.words.protocols.jabber.error import StanzaError
from twisted.internet import defer
from twisted.internet import reactor
from twisted.python import failure
from libervia.backend.core.constants import Const as C
from libervia.backend.tools import xml_tools
from libervia.backend.memory import persistent
from wokkel import disco
from base64 import b64encode
from hashlib import sha1
log = getLogger(__name__)
TIMEOUT = 15
CAP_HASH_ERROR = "ERROR"
class HashGenerationError(Exception):
pass
class ByteIdentity(object):
"""This class manage identity as bytes (needed for i;octet sort), it is used for the hash generation"""
def __init__(self, identity, lang=None):
assert isinstance(identity, disco.DiscoIdentity)
self.category = identity.category.encode("utf-8")
self.idType = identity.type.encode("utf-8")
self.name = identity.name.encode("utf-8") if identity.name else b""
self.lang = lang.encode("utf-8") if lang is not None else b""
def __bytes__(self):
return b"%s/%s/%s/%s" % (self.category, self.idType, self.lang, self.name)
class HashManager(object):
"""map object which manage hashes
persistent storage is update when a new hash is added
"""
def __init__(self, persistent):
self.hashes = {
CAP_HASH_ERROR: disco.DiscoInfo() # used when we can't get disco infos
}
self.persistent = persistent
def __getitem__(self, key):
return self.hashes[key]
def __setitem__(self, hash_, disco_info):
if hash_ in self.hashes:
log.debug("ignoring hash set: it is already known")
return
self.hashes[hash_] = disco_info
self.persistent[hash_] = disco_info.toElement().toXml()
def __contains__(self, hash_):
return self.hashes.__contains__(hash_)
def load(self):
def fill_hashes(hashes):
for hash_, xml in hashes.items():
element = xml_tools.ElementParser()(xml)
disco_info = disco.DiscoInfo.fromElement(element)
for ext_form in disco_info.extensions.values():
# wokkel doesn't call typeCheck on reception, so we do it here
ext_form.typeCheck()
if not disco_info.features and not disco_info.identities:
log.warning(
_(
"no feature/identity found in disco element (hash: {cap_hash}), ignoring: {xml}"
).format(cap_hash=hash_, xml=xml)
)
else:
self.hashes[hash_] = disco_info
log.info("Disco hashes loaded")
d = self.persistent.load()
d.addCallback(fill_hashes)
return d
class Discovery(object):
""" Manage capabilities of entities """
def __init__(self, host):
self.host = host
# TODO: remove legacy hashes
def load(self):
"""Load persistent hashes"""
self.hashes = HashManager(persistent.PersistentDict("disco"))
return self.hashes.load()
@defer.inlineCallbacks
def hasFeature(self, client, feature, jid_=None, node=""):
"""Tell if an entity has the required feature
@param feature: feature namespace
@param jid_: jid of the target, or None for profile's server
@param node(unicode): optional node to use for disco request
@return: a Deferred which fire a boolean (True if feature is available)
"""
disco_infos = yield self.get_infos(client, jid_, node)
defer.returnValue(feature in disco_infos.features)
@defer.inlineCallbacks
def check_feature(self, client, feature, jid_=None, node=""):
"""Like hasFeature, but raise an exception is feature is not Found
@param feature: feature namespace
@param jid_: jid of the target, or None for profile's server
@param node(unicode): optional node to use for disco request
@raise: exceptions.FeatureNotFound
"""
disco_infos = yield self.get_infos(client, jid_, node)
if not feature in disco_infos.features:
raise failure.Failure(exceptions.FeatureNotFound())
@defer.inlineCallbacks
def check_features(self, client, features, jid_=None, identity=None, node=""):
"""Like check_feature, but check several features at once, and check also identity
@param features(iterable[unicode]): features to check
@param jid_(jid.JID): jid of the target, or None for profile's server
@param node(unicode): optional node to use for disco request
@param identity(None, tuple(unicode, unicode): if not None, the entity must have an identity with this (category, type) tuple
@raise: exceptions.FeatureNotFound
"""
disco_infos = yield self.get_infos(client, jid_, node)
if not set(features).issubset(disco_infos.features):
raise failure.Failure(exceptions.FeatureNotFound())
if identity is not None and identity not in disco_infos.identities:
raise failure.Failure(exceptions.FeatureNotFound())
async def has_identity(
self,
client: SatXMPPEntity,
category: str,
type_: str,
jid_: Optional[jid.JID] = None,
node: str = ""
) -> bool:
"""Tell if an entity has the requested identity
@param category: identity category
@param type_: identity type
@param jid_: jid of the target, or None for profile's server
@param node(unicode): optional node to use for disco request
@return: True if the entity has the given identity
"""
disco_infos = await self.get_infos(client, jid_, node)
return (category, type_) in disco_infos.identities
def get_infos(self, client, jid_=None, node="", use_cache=True):
"""get disco infos from jid_, filling capability hash if needed
@param jid_: jid of the target, or None for profile's server
@param node(unicode): optional node to use for disco request
@param use_cache(bool): if True, use cached data if available
@return: a Deferred which fire disco.DiscoInfo
"""
if jid_ is None:
jid_ = jid.JID(client.jid.host)
try:
if not use_cache:
# we ignore cache, so we pretend we haven't found it
raise KeyError
cap_hash = self.host.memory.entity_data_get(
client, jid_, [C.ENTITY_CAP_HASH]
)[C.ENTITY_CAP_HASH]
except (KeyError, exceptions.UnknownEntityError):
# capability hash is not available, we'll compute one
def infos_cb(disco_infos):
cap_hash = self.generate_hash(disco_infos)
for ext_form in disco_infos.extensions.values():
# wokkel doesn't call typeCheck on reception, so we do it here
# to avoid ending up with incorrect types. We have to do it after
# the hash has been generated (str value is needed to compute the
# hash)
ext_form.typeCheck()
self.hashes[cap_hash] = disco_infos
self.host.memory.update_entity_data(
client, jid_, C.ENTITY_CAP_HASH, cap_hash
)
return disco_infos
def infos_eb(fail):
if fail.check(defer.CancelledError):
reason = "request time-out"
fail = failure.Failure(exceptions.TimeOutError(str(fail.value)))
else:
try:
reason = str(fail.value)
except AttributeError:
reason = str(fail)
log.warning(
"can't request disco infos from {jid}: {reason}".format(
jid=jid_.full(), reason=reason
)
)
# XXX we set empty disco in cache, to avoid getting an error or waiting
# for a timeout again the next time
self.host.memory.update_entity_data(
client, jid_, C.ENTITY_CAP_HASH, CAP_HASH_ERROR
)
raise fail
d = client.disco.requestInfo(jid_, nodeIdentifier=node)
d.addCallback(infos_cb)
d.addErrback(infos_eb)
return d
else:
disco_infos = self.hashes[cap_hash]
return defer.succeed(disco_infos)
@defer.inlineCallbacks
def get_items(self, client, jid_=None, node="", use_cache=True):
"""get disco items from jid_, cache them for our own server
@param jid_(jid.JID): jid of the target, or None for profile's server
@param node(unicode): optional node to use for disco request
@param use_cache(bool): if True, use cached data if available
@return: a Deferred which fire disco.DiscoItems
"""
if jid_ is None:
jid_ = client.server_jid
if jid_ == client.server_jid and not node:
# we cache items only for our own server and if node is not set
try:
items = self.host.memory.entity_data_get(
client, jid_, ["DISCO_ITEMS"]
)["DISCO_ITEMS"]
log.debug("[%s] disco items are in cache" % jid_.full())
if not use_cache:
# we ignore cache, so we pretend we haven't found it
raise KeyError
except (KeyError, exceptions.UnknownEntityError):
log.debug("Caching [%s] disco items" % jid_.full())
items = yield client.disco.requestItems(jid_, nodeIdentifier=node)
self.host.memory.update_entity_data(
client, jid_, "DISCO_ITEMS", items
)
else:
try:
items = yield client.disco.requestItems(jid_, nodeIdentifier=node)
except StanzaError as e:
log.warning(
"Error while requesting items for {jid}: {reason}".format(
jid=jid_.full(), reason=e.condition
)
)
items = disco.DiscoItems()
defer.returnValue(items)
def _infos_eb(self, failure_, entity_jid):
failure_.trap(StanzaError)
log.warning(
_("Error while requesting [%(jid)s]: %(error)s")
% {"jid": entity_jid.full(), "error": failure_.getErrorMessage()}
)
def find_service_entity(self, client, category, type_, jid_=None):
"""Helper method to find first available entity from find_service_entities
args are the same as for [find_service_entities]
@return (jid.JID, None): found entity
"""
d = self.host.find_service_entities(client, category, type_)
d.addCallback(lambda entities: entities.pop() if entities else None)
return d
def find_service_entities(self, client, category, type_, jid_=None):
"""Return all available items of an entity which correspond to (category, type_)
@param category: identity's category
@param type_: identitiy's type
@param jid_: the jid of the target server (None for profile's server)
@return: a set of found entities
@raise defer.CancelledError: the request timed out
"""
found_entities = set()
def infos_cb(infos, entity_jid):
if (category, type_) in infos.identities:
found_entities.add(entity_jid)
def got_items(items):
defers_list = []
for item in items:
info_d = self.get_infos(client, item.entity)
info_d.addCallbacks(
infos_cb, self._infos_eb, [item.entity], None, [item.entity]
)
defers_list.append(info_d)
return defer.DeferredList(defers_list)
d = self.get_items(client, jid_)
d.addCallback(got_items)
d.addCallback(lambda __: found_entities)
reactor.callLater(
TIMEOUT, d.cancel
) # FIXME: one bad service make a general timeout
return d
def find_features_set(self, client, features, identity=None, jid_=None):
"""Return entities (including jid_ and its items) offering features
@param features: iterable of features which must be present
@param identity(None, tuple(unicode, unicode)): if not None, accept only this
(category/type) identity
@param jid_: the jid of the target server (None for profile's server)
@param profile: %(doc_profile)s
@return: a set of found entities
"""
if jid_ is None:
jid_ = jid.JID(client.jid.host)
features = set(features)
found_entities = set()
def infos_cb(infos, entity):
if entity is None:
log.warning(_("received an item without jid"))
return
if identity is not None and identity not in infos.identities:
return
if features.issubset(infos.features):
found_entities.add(entity)
def got_items(items):
defer_list = []
for entity in [jid_] + [item.entity for item in items]:
infos_d = self.get_infos(client, entity)
infos_d.addCallbacks(infos_cb, self._infos_eb, [entity], None, [entity])
defer_list.append(infos_d)
return defer.DeferredList(defer_list)
d = self.get_items(client, jid_)
d.addCallback(got_items)
d.addCallback(lambda __: found_entities)
reactor.callLater(
TIMEOUT, d.cancel
) # FIXME: one bad service make a general timeout
return d
def generate_hash(self, services):
""" Generate a unique hash for given service
hash algorithm is the one described in XEP-0115
@param services: iterable of disco.DiscoIdentity/disco.DiscoFeature, as returned by discoHandler.info
"""
s = []
# identities
byte_identities = [
ByteIdentity(service)
for service in services
if isinstance(service, disco.DiscoIdentity)
] # FIXME: lang must be managed here
byte_identities.sort(key=lambda i: i.lang)
byte_identities.sort(key=lambda i: i.idType)
byte_identities.sort(key=lambda i: i.category)
for identity in byte_identities:
s.append(bytes(identity))
s.append(b"<")
# features
byte_features = [
service.encode("utf-8")
for service in services
if isinstance(service, disco.DiscoFeature)
]
byte_features.sort() # XXX: the default sort has the same behaviour as the requested RFC 4790 i;octet sort
for feature in byte_features:
s.append(feature)
s.append(b"<")
# extensions
ext = list(services.extensions.values())
ext.sort(key=lambda f: f.formNamespace.encode('utf-8'))
for extension in ext:
s.append(extension.formNamespace.encode('utf-8'))
s.append(b"<")
fields = extension.fieldList
fields.sort(key=lambda f: f.var.encode('utf-8'))
for field in fields:
s.append(field.var.encode('utf-8'))
s.append(b"<")
values = [v.encode('utf-8') for v in field.values]
values.sort()
for value in values:
s.append(value)
s.append(b"<")
cap_hash = b64encode(sha1(b"".join(s)).digest()).decode('utf-8')
log.debug(_("Capability hash generated: [{cap_hash}]").format(cap_hash=cap_hash))
return cap_hash
@defer.inlineCallbacks
def _disco_infos(
self, entity_jid_s, node="", use_cache=True, profile_key=C.PROF_KEY_NONE
):
"""Discovery method for the bridge
@param entity_jid_s: entity we want to discover
@param use_cache(bool): if True, use cached data if available
@param node(unicode): optional node to use
@return: list of tuples
"""
client = self.host.get_client(profile_key)
entity = jid.JID(entity_jid_s)
disco_infos = yield self.get_infos(client, entity, node, use_cache)
extensions = {}
# FIXME: should extensions be serialised using tools.common.data_format?
for form_type, form in list(disco_infos.extensions.items()):
fields = []
for field in form.fieldList:
data = {"type": field.fieldType}
for attr in ("var", "label", "desc"):
value = getattr(field, attr)
if value is not None:
data[attr] = value
values = [field.value] if field.value is not None else field.values
if field.fieldType == "boolean":
values = [C.bool_const(v) for v in values]
fields.append((data, values))
extensions[form_type or ""] = fields
defer.returnValue((
[str(f) for f in disco_infos.features],
[(cat, type_, name or "")
for (cat, type_), name in list(disco_infos.identities.items())],
extensions))
def items2tuples(self, disco_items):
"""convert disco items to tuple of strings
@param disco_items(iterable[disco.DiscoItem]): items
@return G(tuple[unicode,unicode,unicode]): serialised items
"""
for item in disco_items:
if not item.entity:
log.warning(_("invalid item (no jid)"))
continue
yield (item.entity.full(), item.nodeIdentifier or "", item.name or "")
@defer.inlineCallbacks
def _disco_items(
self, entity_jid_s, node="", use_cache=True, profile_key=C.PROF_KEY_NONE
):
""" Discovery method for the bridge
@param entity_jid_s: entity we want to discover
@param node(unicode): optional node to use
@param use_cache(bool): if True, use cached data if available
@return: list of tuples"""
client = self.host.get_client(profile_key)
entity = jid.JID(entity_jid_s)
disco_items = yield self.get_items(client, entity, node, use_cache)
ret = list(self.items2tuples(disco_items))
defer.returnValue(ret)

View file

@ -0,0 +1,534 @@
#!/usr/bin/env python3
# SAT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import copy
from functools import partial
from typing import Optional
from twisted.words.protocols.jabber import jid
from twisted.internet import defer
from twisted.python import failure
from libervia.backend.core.core_types import EncryptionPlugin, EncryptionSession, MessageData
from libervia.backend.core.i18n import D_, _
from libervia.backend.core.constants import Const as C
from libervia.backend.core import exceptions
from libervia.backend.core.log import getLogger
from libervia.backend.tools.common import data_format
from libervia.backend.tools import utils
from libervia.backend.memory import persistent
log = getLogger(__name__)
class EncryptionHandler:
"""Class to handle encryption sessions for a client"""
plugins = [] # plugin able to encrypt messages
def __init__(self, client):
self.client = client
self._sessions = {} # bare_jid ==> encryption_data
self._stored_session = persistent.PersistentDict(
"core:encryption", profile=client.profile)
@property
def host(self):
return self.client.host_app
async def load_sessions(self):
"""Load persistent sessions"""
await self._stored_session.load()
start_d_list = []
for entity_jid_s, namespace in self._stored_session.items():
entity = jid.JID(entity_jid_s)
start_d_list.append(defer.ensureDeferred(self.start(entity, namespace)))
if start_d_list:
result = await defer.DeferredList(start_d_list)
for idx, (success, err) in enumerate(result):
if not success:
entity_jid_s, namespace = list(self._stored_session.items())[idx]
log.warning(_(
"Could not restart {namespace!r} encryption with {entity}: {err}"
).format(namespace=namespace, entity=entity_jid_s, err=err))
log.info(_("encryption sessions restored"))
@classmethod
def register_plugin(cls, plg_instance, name, namespace, priority=0, directed=False):
"""Register a plugin handling an encryption algorithm
@param plg_instance(object): instance of the plugin
it must have the following methods:
- get_trust_ui(entity): return a XMLUI for trust management
entity(jid.JID): entity to manage
The returned XMLUI must be a form
if may have the following methods:
- start_encryption(entity): start encrypted session
entity(jid.JID): entity to start encrypted session with
- stop_encryption(entity): start encrypted session
entity(jid.JID): entity to stop encrypted session with
if they don't exists, those 2 methods will be ignored.
@param name(unicode): human readable name of the encryption algorithm
@param namespace(unicode): namespace of the encryption algorithm
@param priority(int): priority of this plugin to encrypt an message when not
selected manually
@param directed(bool): True if this plugin is directed (if it works with one
device only at a time)
"""
existing_ns = set()
existing_names = set()
for p in cls.plugins:
existing_ns.add(p.namespace.lower())
existing_names.add(p.name.lower())
if namespace.lower() in existing_ns:
raise exceptions.ConflictError("A plugin with this namespace already exists!")
if name.lower() in existing_names:
raise exceptions.ConflictError("A plugin with this name already exists!")
plugin = EncryptionPlugin(
instance=plg_instance,
name=name,
namespace=namespace,
priority=priority,
directed=directed)
cls.plugins.append(plugin)
cls.plugins.sort(key=lambda p: p.priority)
log.info(_("Encryption plugin registered: {name}").format(name=name))
@classmethod
def getPlugins(cls):
return cls.plugins
@classmethod
def get_plugin(cls, namespace):
try:
return next(p for p in cls.plugins if p.namespace == namespace)
except StopIteration:
raise exceptions.NotFound(_(
"Can't find requested encryption plugin: {namespace}").format(
namespace=namespace))
@classmethod
def get_namespaces(cls):
"""Get available plugin namespaces"""
return {p.namespace for p in cls.getPlugins()}
@classmethod
def get_ns_from_name(cls, name):
"""Retrieve plugin namespace from its name
@param name(unicode): name of the plugin (case insensitive)
@return (unicode): namespace of the plugin
@raise exceptions.NotFound: there is not encryption plugin of this name
"""
for p in cls.plugins:
if p.name.lower() == name.lower():
return p.namespace
raise exceptions.NotFound(_(
"Can't find a plugin with the name \"{name}\".".format(
name=name)))
def get_bridge_data(self, session):
"""Retrieve session data serialized for bridge.
@param session(dict): encryption session
@return (unicode): serialized data for bridge
"""
if session is None:
return ''
plugin = session['plugin']
bridge_data = {'name': plugin.name,
'namespace': plugin.namespace}
if 'directed_devices' in session:
bridge_data['directed_devices'] = session['directed_devices']
return data_format.serialise(bridge_data)
async def _start_encryption(self, plugin, entity):
"""Start encryption with a plugin
This method must be called just before adding a plugin session.
StartEncryptionn method of plugin will be called if it exists.
"""
if not plugin.directed:
await self._stored_session.aset(entity.userhost(), plugin.namespace)
try:
start_encryption = plugin.instance.start_encryption
except AttributeError:
log.debug(f"No start_encryption method found for {plugin.namespace}")
else:
# we copy entity to avoid having the resource changed by stop_encryption
await utils.as_deferred(start_encryption, self.client, copy.copy(entity))
async def _stop_encryption(self, plugin, entity):
"""Stop encryption with a plugin
This method must be called just before removing a plugin session.
StopEncryptionn method of plugin will be called if it exists.
"""
try:
await self._stored_session.adel(entity.userhost())
except KeyError:
pass
try:
stop_encryption = plugin.instance.stop_encryption
except AttributeError:
log.debug(f"No stop_encryption method found for {plugin.namespace}")
else:
# we copy entity to avoid having the resource changed by stop_encryption
return utils.as_deferred(stop_encryption, self.client, copy.copy(entity))
async def start(self, entity, namespace=None, replace=False):
"""Start an encryption session with an entity
@param entity(jid.JID): entity to start an encryption session with
must be bare jid is the algorithm encrypt for all devices
@param namespace(unicode, None): namespace of the encryption algorithm
to use.
None to select automatically an algorithm
@param replace(bool): if True and an encrypted session already exists,
it will be replaced by the new one
"""
if not self.plugins:
raise exceptions.NotFound(_("No encryption plugin is registered, "
"an encryption session can't be started"))
if namespace is None:
plugin = self.plugins[0]
else:
plugin = self.get_plugin(namespace)
bare_jid = entity.userhostJID()
if bare_jid in self._sessions:
# we have already an encryption session with this contact
former_plugin = self._sessions[bare_jid]["plugin"]
if former_plugin.namespace == namespace:
log.info(_("Session with {bare_jid} is already encrypted with {name}. "
"Nothing to do.").format(
bare_jid=bare_jid, name=former_plugin.name))
return
if replace:
# there is a conflict, but replacement is requested
# so we stop previous encryption to use new one
del self._sessions[bare_jid]
await self._stop_encryption(former_plugin, entity)
else:
msg = (_("Session with {bare_jid} is already encrypted with {name}. "
"Please stop encryption session before changing algorithm.")
.format(bare_jid=bare_jid, name=plugin.name))
log.warning(msg)
raise exceptions.ConflictError(msg)
data = {"plugin": plugin}
if plugin.directed:
if not entity.resource:
entity.resource = self.host.memory.main_resource_get(self.client, entity)
if not entity.resource:
raise exceptions.NotFound(
_("No resource found for {destinee}, can't encrypt with {name}")
.format(destinee=entity.full(), name=plugin.name))
log.info(_("No resource specified to encrypt with {name}, using "
"{destinee}.").format(destinee=entity.full(),
name=plugin.name))
# indicate that we encrypt only for some devices
directed_devices = data['directed_devices'] = [entity.resource]
elif entity.resource:
raise ValueError(_("{name} encryption must be used with bare jids."))
await self._start_encryption(plugin, entity)
self._sessions[entity.userhostJID()] = data
log.info(_("Encryption session has been set for {entity_jid} with "
"{encryption_name}").format(
entity_jid=entity.full(), encryption_name=plugin.name))
self.host.bridge.message_encryption_started(
entity.full(),
self.get_bridge_data(data),
self.client.profile)
msg = D_("Encryption session started: your messages with {destinee} are "
"now end to end encrypted using {name} algorithm.").format(
destinee=entity.full(), name=plugin.name)
directed_devices = data.get('directed_devices')
if directed_devices:
msg += "\n" + D_("Message are encrypted only for {nb_devices} device(s): "
"{devices_list}.").format(
nb_devices=len(directed_devices),
devices_list = ', '.join(directed_devices))
self.client.feedback(bare_jid, msg)
async def stop(self, entity, namespace=None):
"""Stop an encryption session with an entity
@param entity(jid.JID): entity with who the encryption session must be stopped
must be bare jid if the algorithm encrypt for all devices
@param namespace(unicode): namespace of the session to stop
when specified, used to check that we stop the right encryption session
"""
session = self.getSession(entity.userhostJID())
if not session:
raise failure.Failure(
exceptions.NotFound(_("There is no encryption session with this "
"entity.")))
plugin = session['plugin']
if namespace is not None and plugin.namespace != namespace:
raise exceptions.InternalError(_(
"The encryption session is not run with the expected plugin: encrypted "
"with {current_name} and was expecting {expected_name}").format(
current_name=session['plugin'].namespace,
expected_name=namespace))
if entity.resource:
try:
directed_devices = session['directed_devices']
except KeyError:
raise exceptions.NotFound(_(
"There is a session for the whole entity (i.e. all devices of the "
"entity), not a directed one. Please use bare jid if you want to "
"stop the whole encryption with this entity."))
try:
directed_devices.remove(entity.resource)
except ValueError:
raise exceptions.NotFound(_("There is no directed session with this "
"entity."))
else:
if not directed_devices:
# if we have no more directed device sessions,
# we stop the whole session
# see comment below for deleting session before stopping encryption
del self._sessions[entity.userhostJID()]
await self._stop_encryption(plugin, entity)
else:
# plugin's stop_encryption may call stop again (that's the case with OTR)
# so we need to remove plugin from session before calling self._stop_encryption
del self._sessions[entity.userhostJID()]
await self._stop_encryption(plugin, entity)
log.info(_("encryption session stopped with entity {entity}").format(
entity=entity.full()))
self.host.bridge.message_encryption_stopped(
entity.full(),
{'name': plugin.name,
'namespace': plugin.namespace,
},
self.client.profile)
msg = D_("Encryption session finished: your messages with {destinee} are "
"NOT end to end encrypted anymore.\nYour server administrators or "
"{destinee} server administrators will be able to read them.").format(
destinee=entity.full())
self.client.feedback(entity, msg)
def getSession(self, entity: jid.JID) -> Optional[EncryptionSession]:
"""Get encryption session for this contact
@param entity(jid.JID): get the session for this entity
must be a bare jid
@return (dict, None): encryption session data
None if there is not encryption for this session with this jid
"""
if entity.resource:
raise ValueError("Full jid given when expecting bare jid")
return self._sessions.get(entity)
def get_namespace(self, entity: jid.JID) -> Optional[str]:
"""Helper method to get the current encryption namespace used
@param entity: get the namespace for this entity must be a bare jid
@return: the algorithm namespace currently used in this session, or None if no
e2ee is currently used.
"""
session = self.getSession(entity)
if session is None:
return None
return session["plugin"].namespace
def get_trust_ui(self, entity_jid, namespace=None):
"""Retrieve encryption UI
@param entity_jid(jid.JID): get the UI for this entity
must be a bare jid
@param namespace(unicode): namespace of the algorithm to manage
if None use current algorithm
@return D(xmlui): XMLUI for trust management
the xmlui is a form
None if there is not encryption for this session with this jid
@raise exceptions.NotFound: no algorithm/plugin found
@raise NotImplementedError: plugin doesn't handle UI management
"""
if namespace is None:
session = self.getSession(entity_jid)
if not session:
raise exceptions.NotFound(
"No encryption session currently active for {entity_jid}"
.format(entity_jid=entity_jid.full()))
plugin = session['plugin']
else:
plugin = self.get_plugin(namespace)
try:
get_trust_ui = plugin.instance.get_trust_ui
except AttributeError:
raise NotImplementedError(
"Encryption plugin doesn't handle trust management UI")
else:
return utils.as_deferred(get_trust_ui, self.client, entity_jid)
## Menus ##
@classmethod
def _import_menus(cls, host):
host.import_menu(
(D_("Encryption"), D_("unencrypted (plain text)")),
partial(cls._on_menu_unencrypted, host=host),
security_limit=0,
help_string=D_("End encrypted session"),
type_=C.MENU_SINGLE,
)
for plg in cls.getPlugins():
host.import_menu(
(D_("Encryption"), plg.name),
partial(cls._on_menu_name, host=host, plg=plg),
security_limit=0,
help_string=D_("Start {name} session").format(name=plg.name),
type_=C.MENU_SINGLE,
)
host.import_menu(
(D_("Encryption"), D_("⛨ {name} trust").format(name=plg.name)),
partial(cls._on_menu_trust, host=host, plg=plg),
security_limit=0,
help_string=D_("Manage {name} trust").format(name=plg.name),
type_=C.MENU_SINGLE,
)
@classmethod
def _on_menu_unencrypted(cls, data, host, profile):
client = host.get_client(profile)
peer_jid = jid.JID(data['jid']).userhostJID()
d = defer.ensureDeferred(client.encryption.stop(peer_jid))
d.addCallback(lambda __: {})
return d
@classmethod
def _on_menu_name(cls, data, host, plg, profile):
client = host.get_client(profile)
peer_jid = jid.JID(data['jid'])
if not plg.directed:
peer_jid = peer_jid.userhostJID()
d = defer.ensureDeferred(
client.encryption.start(peer_jid, plg.namespace, replace=True))
d.addCallback(lambda __: {})
return d
@classmethod
@defer.inlineCallbacks
def _on_menu_trust(cls, data, host, plg, profile):
client = host.get_client(profile)
peer_jid = jid.JID(data['jid']).userhostJID()
ui = yield client.encryption.get_trust_ui(peer_jid, plg.namespace)
defer.returnValue({'xmlui': ui.toXml()})
## Triggers ##
def set_encryption_flag(self, mess_data):
"""Set "encryption" key in mess_data if session with destinee is encrypted"""
to_jid = mess_data['to']
encryption = self._sessions.get(to_jid.userhostJID())
if encryption is not None:
plugin = encryption['plugin']
if mess_data["type"] == "groupchat" and plugin.directed:
raise exceptions.InternalError(
f"encryption flag must not be set for groupchat if encryption algorithm "
f"({encryption['plugin'].name}) is directed!")
mess_data[C.MESS_KEY_ENCRYPTION] = encryption
self.mark_as_encrypted(mess_data, plugin.namespace)
## Misc ##
def mark_as_encrypted(self, mess_data, namespace):
"""Helper method to mark a message as having been e2e encrypted.
This should be used in the post_treat workflow of message_received trigger of
the plugin
@param mess_data(dict): message data as used in post treat workflow
@param namespace(str): namespace of the algorithm used for encrypting the message
"""
mess_data['extra'][C.MESS_KEY_ENCRYPTED] = True
from_bare_jid = mess_data['from'].userhostJID()
if from_bare_jid != self.client.jid.userhostJID():
session = self.getSession(from_bare_jid)
if session is None:
# if we are currently unencrypted, we start a session automatically
# to avoid sending unencrypted messages in an encrypted context
log.info(_(
"Starting e2e session with {peer_jid} as we receive encrypted "
"messages")
.format(peer_jid=from_bare_jid)
)
defer.ensureDeferred(self.start(from_bare_jid, namespace))
return mess_data
def is_encryption_requested(
self,
mess_data: MessageData,
namespace: Optional[str] = None
) -> bool:
"""Helper method to check if encryption is requested in an outgoind message
@param mess_data: message data for outgoing message
@param namespace: if set, check if encryption is requested for the algorithm
specified
@return: True if the encryption flag is present
"""
encryption = mess_data.get(C.MESS_KEY_ENCRYPTION)
if encryption is None:
return False
# we get plugin even if namespace is None to be sure that the key exists
plugin = encryption['plugin']
if namespace is None:
return True
return plugin.namespace == namespace
def isEncrypted(self, mess_data):
"""Helper method to check if a message has the e2e encrypted flag
@param mess_data(dict): message data
@return (bool): True if the encrypted flag is present
"""
return mess_data['extra'].get(C.MESS_KEY_ENCRYPTED, False)
def mark_as_trusted(self, mess_data):
"""Helper methor to mark a message as sent from a trusted entity.
This should be used in the post_treat workflow of message_received trigger of
the plugin
@param mess_data(dict): message data as used in post treat workflow
"""
mess_data[C.MESS_KEY_TRUSTED] = True
return mess_data
def mark_as_untrusted(self, mess_data):
"""Helper methor to mark a message as sent from an untrusted entity.
This should be used in the post_treat workflow of message_received trigger of
the plugin
@param mess_data(dict): message data as used in post treat workflow
"""
mess_data['trusted'] = False
return mess_data

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,3 @@
This directory and subdirectories contains Alembic migration scripts.
Please check Libervia documentation for details.

View file

@ -0,0 +1,89 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = %(here)s
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory.
# prepend_sys_path = .
# timezone to use when rendering the date
# within the migration file as well as the filename.
# string value is passed to dateutil.tz.gettz()
# leave blank for localtime
# timezone =
# max length of characters to apply to the
# "slug" field
# truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; this defaults
# to migration/versions. When using multiple version
# directories, initial revisions must be specified with --version-path
# version_locations = %(here)s/bar %(here)s/bat migration/versions
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
# sqlalchemy.url = driver://user:pass@localhost/dbname
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks = black
# black.type = console_scripts
# black.entrypoint = black
# black.options = -l 79 REVISION_SCRIPT_FILENAME
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

View file

@ -0,0 +1,93 @@
import asyncio
from logging.config import fileConfig
from sqlalchemy import pool
from sqlalchemy.ext.asyncio import create_async_engine
from alembic import context
from libervia.backend.memory import sqla_config
from libervia.backend.memory.sqla_mapping import Base
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = Base.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
db_config = sqla_config.get_db_config()
context.configure(
url=db_config["url"],
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def include_name(name, type_, parent_names):
if type_ == "table":
if name.startswith("pubsub_items_fts"):
return False
return True
def do_run_migrations(connection):
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=True,
include_name=include_name
)
with context.begin_transaction():
context.run_migrations()
async def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
db_config = sqla_config.get_db_config()
engine = create_async_engine(
db_config["url"],
poolclass=pool.NullPool,
future=True,
)
async with engine.connect() as connection:
await connection.run_sync(do_run_migrations)
if context.is_offline_mode():
run_migrations_offline()
else:
asyncio.run(run_migrations_online())

View file

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View file

@ -0,0 +1,49 @@
"""create virtual table for Full-Text Search
Revision ID: 129ac51807e4
Revises: 8974efc51d22
Create Date: 2021-08-13 19:13:54.112538
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '129ac51807e4'
down_revision = '8974efc51d22'
branch_labels = None
depends_on = None
def upgrade():
queries = [
"CREATE VIRTUAL TABLE pubsub_items_fts "
"USING fts5(data, content=pubsub_items, content_rowid=id)",
"CREATE TRIGGER pubsub_items_fts_sync_ins AFTER INSERT ON pubsub_items BEGIN"
" INSERT INTO pubsub_items_fts(rowid, data) VALUES (new.id, new.data);"
"END",
"CREATE TRIGGER pubsub_items_fts_sync_del AFTER DELETE ON pubsub_items BEGIN"
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) "
"VALUES('delete', old.id, old.data);"
"END",
"CREATE TRIGGER pubsub_items_fts_sync_upd AFTER UPDATE ON pubsub_items BEGIN"
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) VALUES"
"('delete', old.id, old.data);"
" INSERT INTO pubsub_items_fts(rowid, data) VALUES(new.id, new.data);"
"END",
"INSERT INTO pubsub_items_fts(rowid, data) SELECT id, data from pubsub_items"
]
for q in queries:
op.execute(sa.DDL(q))
def downgrade():
queries = [
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_ins",
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_del",
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_upd",
"DROP TABLE IF EXISTS pubsub_items_fts",
]
for q in queries:
op.execute(sa.DDL(q))

View file

@ -0,0 +1,60 @@
"""add origin_id column to history and adapt constraints
Revision ID: 4b002773cf92
Revises: 79e5f3313fa4
Create Date: 2022-06-13 16:10:39.711634
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '4b002773cf92'
down_revision = '79e5f3313fa4'
branch_labels = None
depends_on = None
def upgrade():
with op.batch_alter_table('history', schema=None) as batch_op:
batch_op.add_column(sa.Column('origin_id', sa.Text(), nullable=True))
batch_op.create_unique_constraint('uq_origin_id', ['profile_id', 'origin_id', 'source'])
with op.batch_alter_table('message', schema=None) as batch_op:
batch_op.alter_column('history_uid',
existing_type=sa.TEXT(),
nullable=False)
batch_op.alter_column('message',
existing_type=sa.TEXT(),
nullable=False)
with op.batch_alter_table('subject', schema=None) as batch_op:
batch_op.alter_column('history_uid',
existing_type=sa.TEXT(),
nullable=False)
batch_op.alter_column('subject',
existing_type=sa.TEXT(),
nullable=False)
def downgrade():
with op.batch_alter_table('subject', schema=None) as batch_op:
batch_op.alter_column('subject',
existing_type=sa.TEXT(),
nullable=True)
batch_op.alter_column('history_uid',
existing_type=sa.TEXT(),
nullable=True)
with op.batch_alter_table('message', schema=None) as batch_op:
batch_op.alter_column('message',
existing_type=sa.TEXT(),
nullable=True)
batch_op.alter_column('history_uid',
existing_type=sa.TEXT(),
nullable=True)
with op.batch_alter_table('history', schema=None) as batch_op:
batch_op.drop_constraint('uq_origin_id', type_='unique')
batch_op.drop_column('origin_id')

View file

@ -0,0 +1,410 @@
"""drop message_types table + fix nullable + rename constraints
Revision ID: 602caf848068
Revises:
Create Date: 2021-06-26 12:42:54.148313
"""
from alembic import op
from sqlalchemy import (
Table,
Column,
MetaData,
TEXT,
INTEGER,
Text,
Integer,
Float,
Enum,
ForeignKey,
Index,
PrimaryKeyConstraint,
)
from sqlalchemy.sql import table, column
# revision identifiers, used by Alembic.
revision = "602caf848068"
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
# we have to recreate former tables for batch_alter_table's reflexion, otherwise the
# database will be used, and this will keep unammed UNIQUE constraints in addition
# to the named ones that we create
metadata = MetaData(
naming_convention={
"ix": "ix_%(column_0_label)s",
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s",
},
)
old_profiles_table = Table(
"profiles",
metadata,
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
Column("name", Text, unique=True),
)
old_components_table = Table(
"components",
metadata,
Column(
"profile_id",
ForeignKey("profiles.id", ondelete="CASCADE"),
nullable=True,
primary_key=True,
),
Column("entry_point", Text, nullable=False),
)
old_message_table = Table(
"message",
metadata,
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
Column("message", Text),
Column("language", Text),
Index("message__history_uid", "history_uid"),
)
old_subject_table = Table(
"subject",
metadata,
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
Column("subject", Text),
Column("language", Text),
Index("subject__history_uid", "history_uid"),
)
old_thread_table = Table(
"thread",
metadata,
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
Column("thread_id", Text),
Column("parent_id", Text),
Index("thread__history_uid", "history_uid"),
)
old_history_table = Table(
"history",
metadata,
Column("uid", Text, primary_key=True, nullable=True),
Column("stanza_id", Text),
Column("update_uid", Text),
Column("profile_id", Integer, ForeignKey("profiles.id", ondelete="CASCADE")),
Column("source", Text),
Column("dest", Text),
Column("source_res", Text),
Column("dest_res", Text),
Column("timestamp", Float, nullable=False),
Column("received_timestamp", Float),
Column("type", Text, ForeignKey("message_types.type")),
Column("extra", Text),
Index("history__profile_id_timestamp", "profile_id", "timestamp"),
Index(
"history__profile_id_received_timestamp", "profile_id", "received_timestamp"
),
)
old_param_gen_table = Table(
"param_gen",
metadata,
Column("category", Text, primary_key=True),
Column("name", Text, primary_key=True),
Column("value", Text),
)
old_param_ind_table = Table(
"param_ind",
metadata,
Column("category", Text, primary_key=True),
Column("name", Text, primary_key=True),
Column(
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
),
Column("value", Text),
)
old_private_gen_table = Table(
"private_gen",
metadata,
Column("namespace", Text, primary_key=True),
Column("key", Text, primary_key=True),
Column("value", Text),
)
old_private_ind_table = Table(
"private_ind",
metadata,
Column("namespace", Text, primary_key=True),
Column("key", Text, primary_key=True),
Column(
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
),
Column("value", Text),
)
old_private_gen_bin_table = Table(
"private_gen_bin",
metadata,
Column("namespace", Text, primary_key=True),
Column("key", Text, primary_key=True),
Column("value", Text),
)
old_private_ind_bin_table = Table(
"private_ind_bin",
metadata,
Column("namespace", Text, primary_key=True),
Column("key", Text, primary_key=True),
Column(
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
),
Column("value", Text),
)
old_files_table = Table(
"files",
metadata,
Column("id", Text, primary_key=True),
Column("public_id", Text, unique=True),
Column("version", Text, primary_key=True),
Column("parent", Text, nullable=False),
Column(
"type",
Enum("file", "directory", name="file_type", create_constraint=True),
nullable=False,
server_default="file",
),
Column("file_hash", Text),
Column("hash_algo", Text),
Column("name", Text, nullable=False),
Column("size", Integer),
Column("namespace", Text),
Column("media_type", Text),
Column("media_subtype", Text),
Column("created", Float, nullable=False),
Column("modified", Float),
Column("owner", Text),
Column("access", Text),
Column("extra", Text),
Column("profile_id", ForeignKey("profiles.id", ondelete="CASCADE")),
Index("files__profile_id_owner_parent", "profile_id", "owner", "parent"),
Index(
"files__profile_id_owner_media_type_media_subtype",
"profile_id",
"owner",
"media_type",
"media_subtype",
),
)
op.drop_table("message_types")
with op.batch_alter_table(
"profiles", copy_from=old_profiles_table, schema=None
) as batch_op:
batch_op.create_unique_constraint(batch_op.f("uq_profiles_name"), ["name"])
with op.batch_alter_table(
"components",
copy_from=old_components_table,
naming_convention={
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
},
schema=None,
) as batch_op:
batch_op.create_unique_constraint(batch_op.f("uq_profiles_name"), ["name"])
with op.batch_alter_table(
"history",
copy_from=old_history_table,
naming_convention={
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
},
schema=None,
) as batch_op:
batch_op.alter_column("uid", existing_type=TEXT(), nullable=False)
batch_op.alter_column(
"type",
type_=Enum(
"chat",
"error",
"groupchat",
"headline",
"normal",
"info",
name="message_type",
create_constraint=True,
),
existing_type=TEXT(),
nullable=False,
)
batch_op.create_unique_constraint(
batch_op.f("uq_history_profile_id"),
["profile_id", "stanza_id", "source", "dest"],
)
batch_op.drop_constraint("fk_history_type_message_types", type_="foreignkey")
with op.batch_alter_table(
"message", copy_from=old_message_table, schema=None
) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
)
with op.batch_alter_table(
"subject", copy_from=old_subject_table, schema=None
) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
)
with op.batch_alter_table(
"thread", copy_from=old_thread_table, schema=None
) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
)
with op.batch_alter_table(
"param_gen", copy_from=old_param_gen_table, schema=None
) as batch_op:
batch_op.alter_column("category", existing_type=TEXT(), nullable=False)
batch_op.alter_column("name", existing_type=TEXT(), nullable=False)
with op.batch_alter_table(
"param_ind", copy_from=old_param_ind_table, schema=None
) as batch_op:
batch_op.alter_column("category", existing_type=TEXT(), nullable=False)
batch_op.alter_column("name", existing_type=TEXT(), nullable=False)
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
with op.batch_alter_table(
"private_gen", copy_from=old_private_gen_table, schema=None
) as batch_op:
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
with op.batch_alter_table(
"private_ind", copy_from=old_private_ind_table, schema=None
) as batch_op:
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
with op.batch_alter_table(
"private_gen_bin", copy_from=old_private_gen_bin_table, schema=None
) as batch_op:
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
# found some invalid rows in local database, maybe old values made during development,
# but in doubt we have to delete them
op.execute("DELETE FROM private_ind_bin WHERE namespace IS NULL")
with op.batch_alter_table(
"private_ind_bin", copy_from=old_private_ind_bin_table, schema=None
) as batch_op:
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
with op.batch_alter_table(
"files", copy_from=old_files_table, schema=None
) as batch_op:
batch_op.create_unique_constraint(batch_op.f("uq_files_public_id"), ["public_id"])
batch_op.alter_column(
"type",
type_=Enum("file", "directory", name="file_type", create_constraint=True),
existing_type=Text(),
nullable=False,
)
def downgrade():
# downgrade doesn't restore the exact same state as before upgrade, as it
# would be useless and waste of resource to restore broken things such as
# anonymous constraints
with op.batch_alter_table("thread", schema=None) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
)
with op.batch_alter_table("subject", schema=None) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
)
with op.batch_alter_table("private_ind_bin", schema=None) as batch_op:
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("private_ind", schema=None) as batch_op:
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("private_gen_bin", schema=None) as batch_op:
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("private_gen", schema=None) as batch_op:
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("param_ind", schema=None) as batch_op:
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
batch_op.alter_column("name", existing_type=TEXT(), nullable=True)
batch_op.alter_column("category", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("param_gen", schema=None) as batch_op:
batch_op.alter_column("name", existing_type=TEXT(), nullable=True)
batch_op.alter_column("category", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("message", schema=None) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
)
op.create_table(
"message_types",
Column("type", TEXT(), nullable=True),
PrimaryKeyConstraint("type"),
)
message_types_table = table("message_types", column("type", TEXT()))
op.bulk_insert(
message_types_table,
[
{"type": "chat"},
{"type": "error"},
{"type": "groupchat"},
{"type": "headline"},
{"type": "normal"},
{"type": "info"},
],
)
with op.batch_alter_table("history", schema=None) as batch_op:
batch_op.alter_column(
"type",
type_=TEXT(),
existing_type=TEXT(),
nullable=True,
)
batch_op.create_foreign_key(
batch_op.f("fk_history_type_message_types"),
"message_types",
["type"],
["type"],
)
batch_op.alter_column("uid", existing_type=TEXT(), nullable=True)

View file

@ -0,0 +1,33 @@
"""create table for pubsub subscriptions
Revision ID: 79e5f3313fa4
Revises: 129ac51807e4
Create Date: 2022-03-14 17:15:00.689871
"""
from alembic import op
import sqlalchemy as sa
from libervia.backend.memory.sqla_mapping import JID
# revision identifiers, used by Alembic.
revision = '79e5f3313fa4'
down_revision = '129ac51807e4'
branch_labels = None
depends_on = None
def upgrade():
op.create_table('pubsub_subs',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('node_id', sa.Integer(), nullable=False),
sa.Column('subscriber', JID(), nullable=True),
sa.Column('state', sa.Enum('SUBSCRIBED', 'PENDING', name='state'), nullable=True),
sa.ForeignKeyConstraint(['node_id'], ['pubsub_nodes.id'], name=op.f('fk_pubsub_subs_node_id_pubsub_nodes'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id', name=op.f('pk_pubsub_subs')),
sa.UniqueConstraint('node_id', 'subscriber', name=op.f('uq_pubsub_subs_node_id'))
)
def downgrade():
op.drop_table('pubsub_subs')

View file

@ -0,0 +1,57 @@
"""create tables for Pubsub caching
Revision ID: 8974efc51d22
Revises: 602caf848068
Create Date: 2021-07-27 16:38:54.658212
"""
from alembic import op
import sqlalchemy as sa
from libervia.backend.memory.sqla_mapping import JID, Xml
# revision identifiers, used by Alembic.
revision = '8974efc51d22'
down_revision = '602caf848068'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('pubsub_nodes',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('profile_id', sa.Integer(), nullable=True),
sa.Column('service', JID(), nullable=True),
sa.Column('name', sa.Text(), nullable=False),
sa.Column('subscribed', sa.Boolean(create_constraint=True, name='subscribed_bool'), nullable=False),
sa.Column('analyser', sa.Text(), nullable=True),
sa.Column('sync_state', sa.Enum('IN_PROGRESS', 'COMPLETED', 'ERROR', 'NO_SYNC', name='sync_state', create_constraint=True), nullable=True),
sa.Column('sync_state_updated', sa.Float(), nullable=False),
sa.Column('type', sa.Text(), nullable=True),
sa.Column('subtype', sa.Text(), nullable=True),
sa.Column('extra', sa.JSON(), nullable=True),
sa.ForeignKeyConstraint(['profile_id'], ['profiles.id'], name=op.f('fk_pubsub_nodes_profile_id_profiles'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id', name=op.f('pk_pubsub_nodes')),
sa.UniqueConstraint('profile_id', 'service', 'name', name=op.f('uq_pubsub_nodes_profile_id'))
)
op.create_table('pubsub_items',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('node_id', sa.Integer(), nullable=False),
sa.Column('name', sa.Text(), nullable=False),
sa.Column('data', Xml(), nullable=False),
sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
sa.Column('updated', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
sa.Column('parsed', sa.JSON(), nullable=True),
sa.ForeignKeyConstraint(['node_id'], ['pubsub_nodes.id'], name=op.f('fk_pubsub_items_node_id_pubsub_nodes'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id', name=op.f('pk_pubsub_items')),
sa.UniqueConstraint('node_id', 'name', name=op.f('uq_pubsub_items_node_id'))
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('pubsub_items')
op.drop_table('pubsub_nodes')
# ### end Alembic commands ###

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,317 @@
#!/usr/bin/env python3
# SAT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from twisted.internet import defer
from twisted.python import failure
from libervia.backend.core.i18n import _
from libervia.backend.core.log import getLogger
log = getLogger(__name__)
class MemoryNotInitializedError(Exception):
pass
class PersistentDict:
r"""A dictionary which save persistently each value assigned
/!\ be careful, each assignment means a database write
/!\ Memory must be initialised before loading/setting value with instances of this class"""
storage = None
binary = False
def __init__(self, namespace, profile=None):
"""
@param namespace: unique namespace for this dictionary
@param profile(unicode, None): profile which *MUST* exists, or None for general values
"""
if not self.storage:
log.error(_("PersistentDict can't be used before memory initialisation"))
raise MemoryNotInitializedError
self._cache = {}
self.namespace = namespace
self.profile = profile
def _set_cache(self, data):
self._cache = data
def load(self):
"""Load persistent data from storage.
need to be called before any other operation
@return: defers the PersistentDict instance itself
"""
d = defer.ensureDeferred(self.storage.get_privates(
self.namespace, binary=self.binary, profile=self.profile
))
d.addCallback(self._set_cache)
d.addCallback(lambda __: self)
return d
def iteritems(self):
return iter(self._cache.items())
def items(self):
return self._cache.items()
def __repr__(self):
return self._cache.__repr__()
def __str__(self):
return self._cache.__str__()
def __lt__(self, other):
return self._cache.__lt__(other)
def __le__(self, other):
return self._cache.__le__(other)
def __eq__(self, other):
return self._cache.__eq__(other)
def __ne__(self, other):
return self._cache.__ne__(other)
def __gt__(self, other):
return self._cache.__gt__(other)
def __ge__(self, other):
return self._cache.__ge__(other)
def __cmp__(self, other):
return self._cache.__cmp__(other)
def __hash__(self):
return self._cache.__hash__()
def __bool__(self):
return self._cache.__len__() != 0
def __contains__(self, key):
return self._cache.__contains__(key)
def __iter__(self):
return self._cache.__iter__()
def __getitem__(self, key):
return self._cache.__getitem__(key)
def __setitem__(self, key, value):
defer.ensureDeferred(
self.storage.set_private_value(
self.namespace, key, value, self.binary, self.profile
)
)
return self._cache.__setitem__(key, value)
def __delitem__(self, key):
self.storage.del_private_value(self.namespace, key, self.binary, self.profile)
return self._cache.__delitem__(key)
def clear(self):
"""Delete all values from this namespace"""
self._cache.clear()
return self.storage.del_private_namespace(self.namespace, self.binary, self.profile)
def get(self, key, default=None):
return self._cache.get(key, default)
def aset(self, key, value):
"""Async set, return a Deferred fired when value is actually stored"""
self._cache.__setitem__(key, value)
return defer.ensureDeferred(
self.storage.set_private_value(
self.namespace, key, value, self.binary, self.profile
)
)
def adel(self, key):
"""Async del, return a Deferred fired when value is actually deleted"""
self._cache.__delitem__(key)
return self.storage.del_private_value(
self.namespace, key, self.binary, self.profile)
def setdefault(self, key, default):
try:
return self._cache[key]
except:
self.__setitem__(key, default)
return default
def force(self, name):
"""Force saving of an attribute to storage
@return: deferred fired when data is actually saved
"""
return defer.ensureDeferred(
self.storage.set_private_value(
self.namespace, name, self._cache[name], self.binary, self.profile
)
)
class PersistentBinaryDict(PersistentDict):
"""Persistent dict where value can be any python data (instead of string only)"""
binary = True
class LazyPersistentBinaryDict(PersistentBinaryDict):
r"""PersistentBinaryDict which get key/value when needed
This Persistent need more database access, it is suitable for largest data,
to save memory.
/!\ most of methods return a Deferred
"""
# TODO: missing methods should be implemented using database access
# TODO: a cache would be useful (which is deleted after a timeout)
def load(self):
# we show a warning as calling load on LazyPersistentBinaryDict sounds like a code mistake
log.warning(_("Calling load on LazyPersistentBinaryDict while it's not needed"))
def iteritems(self):
raise NotImplementedError
def items(self):
d = defer.ensureDeferred(self.storage.get_privates(
self.namespace, binary=self.binary, profile=self.profile
))
d.addCallback(lambda data_dict: data_dict.items())
return d
def all(self):
return defer.ensureDeferred(self.storage.get_privates(
self.namespace, binary=self.binary, profile=self.profile
))
def __repr__(self):
return self.__str__()
def __str__(self):
return "LazyPersistentBinaryDict (namespace: {})".format(self.namespace)
def __lt__(self, other):
raise NotImplementedError
def __le__(self, other):
raise NotImplementedError
def __eq__(self, other):
raise NotImplementedError
def __ne__(self, other):
raise NotImplementedError
def __gt__(self, other):
raise NotImplementedError
def __ge__(self, other):
raise NotImplementedError
def __cmp__(self, other):
raise NotImplementedError
def __hash__(self):
return hash(str(self.__class__) + self.namespace + (self.profile or ''))
def __bool__(self):
raise NotImplementedError
def __contains__(self, key):
raise NotImplementedError
def __iter__(self):
raise NotImplementedError
def _data2value(self, data, key):
try:
return data[key]
except KeyError as e:
# we return a Failure here to avoid the jump
# into debugger in debug mode.
raise failure.Failure(e)
def __getitem__(self, key):
"""get the value as a Deferred"""
d = defer.ensureDeferred(self.storage.get_privates(
self.namespace, keys=[key], binary=self.binary, profile=self.profile
))
d.addCallback(self._data2value, key)
return d
def __setitem__(self, key, value):
defer.ensureDeferred(
self.storage.set_private_value(
self.namespace, key, value, self.binary, self.profile
)
)
def __delitem__(self, key):
self.storage.del_private_value(self.namespace, key, self.binary, self.profile)
def _default_or_exception(self, failure_, default):
failure_.trap(KeyError)
return default
def get(self, key, default=None):
d = self.__getitem__(key)
d.addErrback(self._default_or_exception, default=default)
return d
def aset(self, key, value):
"""Async set, return a Deferred fired when value is actually stored"""
# FIXME: redundant with force, force must be removed
# XXX: similar as PersistentDict.aset, but doesn't use cache
return defer.ensureDeferred(
self.storage.set_private_value(
self.namespace, key, value, self.binary, self.profile
)
)
def adel(self, key):
"""Async del, return a Deferred fired when value is actually deleted"""
# XXX: similar as PersistentDict.adel, but doesn't use cache
return self.storage.del_private_value(
self.namespace, key, self.binary, self.profile)
def setdefault(self, key, default):
raise NotImplementedError
def force(self, name, value):
"""Force saving of an attribute to storage
@param value(object): value is needed for LazyPersistentBinaryDict
@return: deferred fired when data is actually saved
"""
return defer.ensureDeferred(
self.storage.set_private_value(
self.namespace, name, value, self.binary, self.profile
)
)
def remove(self, key):
"""Delete a key from sotrage, and return a deferred called when it's done
@param key(unicode): key to delete
@return (D): A deferred fired when delete is done
"""
return self.storage.del_private_value(self.namespace, key, self.binary, self.profile)

File diff suppressed because it is too large Load diff

Some files were not shown because too many files have changed in this diff Show more