refactoring: renamed sat package to libervia.backend
This commit is contained in:
parent
a15648d682
commit
d2685ba11b
22 changed files with 1293 additions and 1293 deletions
0
libervia/backend/memory/__init__.py
Normal file
0
libervia/backend/memory/__init__.py
Normal file
281
libervia/backend/memory/cache.py
Normal file
281
libervia/backend/memory/cache.py
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
|
||||
# SAT: a jabber client
|
||||
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from io import BufferedIOBase
|
||||
import mimetypes
|
||||
from pathlib import Path
|
||||
import pickle as pickle
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from libervia.backend.core import exceptions
|
||||
from libervia.backend.core.constants import Const as C
|
||||
from libervia.backend.core.i18n import _
|
||||
from libervia.backend.core.log import getLogger
|
||||
from libervia.backend.tools.common import regex
|
||||
|
||||
|
||||
log = getLogger(__name__)
|
||||
|
||||
DEFAULT_EXT = ".raw"
|
||||
|
||||
|
||||
class Cache(object):
|
||||
"""generic file caching"""
|
||||
|
||||
def __init__(self, host, profile):
|
||||
"""
|
||||
@param profile(unicode, None): name of the profile to set the cache for
|
||||
if None, the cache will be common for all profiles
|
||||
"""
|
||||
self.profile = profile
|
||||
path_elts = [host.memory.config_get("", "local_dir"), C.CACHE_DIR]
|
||||
if profile:
|
||||
path_elts.extend(["profiles", regex.path_escape(profile)])
|
||||
else:
|
||||
path_elts.append("common")
|
||||
self.cache_dir = Path(*path_elts)
|
||||
|
||||
self.cache_dir.mkdir(0o700, parents=True, exist_ok=True)
|
||||
self.purge()
|
||||
|
||||
def purge(self):
|
||||
# remove expired files from cache
|
||||
# TODO: this should not be called only on startup, but at regular interval
|
||||
# (e.g. once a day)
|
||||
purged = set()
|
||||
# we sort files to have metadata files first
|
||||
for cache_file in sorted(self.cache_dir.iterdir()):
|
||||
if cache_file in purged:
|
||||
continue
|
||||
try:
|
||||
with cache_file.open('rb') as f:
|
||||
cache_data = pickle.load(f)
|
||||
except IOError:
|
||||
log.warning(
|
||||
_("Can't read metadata file at {path}")
|
||||
.format(path=cache_file))
|
||||
continue
|
||||
except (pickle.UnpicklingError, EOFError):
|
||||
log.debug(f"File at {cache_file} is not a metadata file")
|
||||
continue
|
||||
try:
|
||||
eol = cache_data['eol']
|
||||
filename = cache_data['filename']
|
||||
except KeyError:
|
||||
log.warning(
|
||||
_("Invalid cache metadata at {path}")
|
||||
.format(path=cache_file))
|
||||
continue
|
||||
|
||||
filepath = self.getPath(filename)
|
||||
|
||||
if not filepath.exists():
|
||||
log.warning(_(
|
||||
"cache {cache_file!r} references an inexisting file: {filepath!r}"
|
||||
).format(cache_file=str(cache_file), filepath=str(filepath)))
|
||||
log.debug("purging cache with missing file")
|
||||
cache_file.unlink()
|
||||
elif eol < time.time():
|
||||
log.debug(
|
||||
"purging expired cache {filepath!r} (expired for {time}s)"
|
||||
.format(filepath=str(filepath), time=int(time.time() - eol))
|
||||
)
|
||||
cache_file.unlink()
|
||||
try:
|
||||
filepath.unlink()
|
||||
except FileNotFoundError:
|
||||
log.warning(
|
||||
_("following file is missing while purging cache: {path}")
|
||||
.format(path=filepath)
|
||||
)
|
||||
purged.add(cache_file)
|
||||
purged.add(filepath)
|
||||
|
||||
def getPath(self, filename: str) -> Path:
|
||||
"""return cached file URL
|
||||
|
||||
@param filename: cached file name (cache data or actual file)
|
||||
@return: path to the cached file
|
||||
"""
|
||||
if not filename or "/" in filename:
|
||||
log.error(
|
||||
"invalid char found in file name, hack attempt? name:{}".format(filename)
|
||||
)
|
||||
raise exceptions.DataError("Invalid char found")
|
||||
return self.cache_dir / filename
|
||||
|
||||
def get_metadata(self, uid: str, update_eol: bool = True) -> Optional[Dict[str, Any]]:
|
||||
"""Retrieve metadata for cached data
|
||||
|
||||
@param uid(unicode): unique identifier of file
|
||||
@param update_eol(bool): True if eol must extended
|
||||
if True, max_age will be added to eol (only if it is not already expired)
|
||||
@return (dict, None): metadata with following keys:
|
||||
see [cache_data] for data details, an additional "path" key is the full path to
|
||||
cached file.
|
||||
None if file is not in cache (or cache is invalid)
|
||||
"""
|
||||
|
||||
uid = uid.strip()
|
||||
if not uid:
|
||||
raise exceptions.InternalError("uid must not be empty")
|
||||
cache_url = self.getPath(uid)
|
||||
if not cache_url.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with cache_url.open("rb") as f:
|
||||
cache_data = pickle.load(f)
|
||||
except (IOError, EOFError) as e:
|
||||
log.warning(f"can't read cache at {cache_url}: {e}")
|
||||
return None
|
||||
except pickle.UnpicklingError:
|
||||
log.warning(f"invalid cache found at {cache_url}")
|
||||
return None
|
||||
|
||||
try:
|
||||
eol = cache_data["eol"]
|
||||
except KeyError:
|
||||
log.warning("no End Of Life found for cached file {}".format(uid))
|
||||
eol = 0
|
||||
if eol < time.time():
|
||||
log.debug(
|
||||
"removing expired cache (expired for {}s)".format(time.time() - eol)
|
||||
)
|
||||
return None
|
||||
|
||||
if update_eol:
|
||||
try:
|
||||
max_age = cache_data["max_age"]
|
||||
except KeyError:
|
||||
log.warning(f"no max_age found for cache at {cache_url}, using default")
|
||||
max_age = cache_data["max_age"] = C.DEFAULT_MAX_AGE
|
||||
now = int(time.time())
|
||||
cache_data["last_access"] = now
|
||||
cache_data["eol"] = now + max_age
|
||||
with cache_url.open("wb") as f:
|
||||
pickle.dump(cache_data, f, protocol=2)
|
||||
|
||||
cache_data["path"] = self.getPath(cache_data["filename"])
|
||||
return cache_data
|
||||
|
||||
def get_file_path(self, uid: str) -> Path:
|
||||
"""Retrieve absolute path to file
|
||||
|
||||
@param uid(unicode): unique identifier of file
|
||||
@return (unicode, None): absolute path to cached file
|
||||
None if file is not in cache (or cache is invalid)
|
||||
"""
|
||||
metadata = self.get_metadata(uid)
|
||||
if metadata is not None:
|
||||
return metadata["path"]
|
||||
|
||||
def remove_from_cache(self, uid, metadata=None):
|
||||
"""Remove data from cache
|
||||
|
||||
@param uid(unicode): unique identifier cache file
|
||||
"""
|
||||
cache_data = self.get_metadata(uid, update_eol=False)
|
||||
if cache_data is None:
|
||||
log.debug(f"cache with uid {uid!r} has already expired or been removed")
|
||||
return
|
||||
|
||||
try:
|
||||
filename = cache_data['filename']
|
||||
except KeyError:
|
||||
log.warning(_("missing filename for cache {uid!r}") .format(uid=uid))
|
||||
else:
|
||||
filepath = self.getPath(filename)
|
||||
try:
|
||||
filepath.unlink()
|
||||
except FileNotFoundError:
|
||||
log.warning(
|
||||
_("missing file referenced in cache {uid!r}: {filename}")
|
||||
.format(uid=uid, filename=filename)
|
||||
)
|
||||
|
||||
cache_file = self.getPath(uid)
|
||||
cache_file.unlink()
|
||||
log.debug(f"cache with uid {uid!r} has been removed")
|
||||
|
||||
def cache_data(
|
||||
self,
|
||||
source: str,
|
||||
uid: str,
|
||||
mime_type: Optional[str] = None,
|
||||
max_age: Optional[int] = None,
|
||||
original_filename: Optional[str] = None
|
||||
) -> BufferedIOBase:
|
||||
"""create cache metadata and file object to use for actual data
|
||||
|
||||
@param source: source of the cache (should be plugin's import_name)
|
||||
@param uid: an identifier of the file which must be unique
|
||||
@param mime_type: MIME type of the file to cache
|
||||
it will be used notably to guess file extension
|
||||
It may be autogenerated if filename is specified
|
||||
@param max_age: maximum age in seconds
|
||||
the cache metadata will have an "eol" (end of life)
|
||||
None to use default value
|
||||
0 to ignore cache (file will be re-downloaded on each access)
|
||||
@param original_filename: if not None, will be used to retrieve file extension and
|
||||
guess
|
||||
mime type, and stored in "original_filename"
|
||||
@return: file object opened in write mode
|
||||
you have to close it yourself (hint: use ``with`` statement)
|
||||
"""
|
||||
if max_age is None:
|
||||
max_age = C.DEFAULT_MAX_AGE
|
||||
cache_data = {
|
||||
"source": source,
|
||||
# we also store max_age for updating eol
|
||||
"max_age": max_age,
|
||||
}
|
||||
cache_url = self.getPath(uid)
|
||||
if original_filename is not None:
|
||||
cache_data["original_filename"] = original_filename
|
||||
if mime_type is None:
|
||||
# we have original_filename but not MIME type, we try to guess the later
|
||||
mime_type = mimetypes.guess_type(original_filename, strict=False)[0]
|
||||
if mime_type:
|
||||
ext = mimetypes.guess_extension(mime_type, strict=False)
|
||||
if ext is None:
|
||||
log.warning(
|
||||
"can't find extension for MIME type {}".format(mime_type)
|
||||
)
|
||||
ext = DEFAULT_EXT
|
||||
elif ext == ".jpe":
|
||||
ext = ".jpg"
|
||||
else:
|
||||
ext = DEFAULT_EXT
|
||||
mime_type = None
|
||||
filename = uid + ext
|
||||
now = int(time.time())
|
||||
cache_data.update({
|
||||
"filename": filename,
|
||||
"creation": now,
|
||||
"eol": now + max_age,
|
||||
"mime_type": mime_type,
|
||||
})
|
||||
file_path = self.getPath(filename)
|
||||
|
||||
with open(cache_url, "wb") as f:
|
||||
pickle.dump(cache_data, f, protocol=2)
|
||||
|
||||
return file_path.open("wb")
|
||||
170
libervia/backend/memory/crypto.py
Normal file
170
libervia/backend/memory/crypto.py
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# SAT: a jabber client
|
||||
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
|
||||
# Copyright (C) 2013-2016 Adrien Cossa (souliane@mailoo.org)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from os import urandom
|
||||
from base64 import b64encode, b64decode
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
|
||||
|
||||
crypto_backend = default_backend()
|
||||
|
||||
|
||||
class BlockCipher:
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
MAX_KEY_SIZE = 32
|
||||
IV_SIZE = BLOCK_SIZE # initialization vector size, 16 bits
|
||||
|
||||
@staticmethod
|
||||
def encrypt(key, text, leave_empty=True):
|
||||
"""Encrypt a message.
|
||||
|
||||
Based on http://stackoverflow.com/a/12525165
|
||||
|
||||
@param key (unicode): the encryption key
|
||||
@param text (unicode): the text to encrypt
|
||||
@param leave_empty (bool): if True, empty text will be returned "as is"
|
||||
@return (D(str)): base-64 encoded encrypted message
|
||||
"""
|
||||
if leave_empty and text == "":
|
||||
return ""
|
||||
iv = BlockCipher.get_random_key()
|
||||
key = key.encode()
|
||||
key = (
|
||||
key[: BlockCipher.MAX_KEY_SIZE]
|
||||
if len(key) >= BlockCipher.MAX_KEY_SIZE
|
||||
else BlockCipher.pad(key)
|
||||
)
|
||||
|
||||
cipher = Cipher(algorithms.AES(key), modes.CFB8(iv), backend=crypto_backend)
|
||||
encryptor = cipher.encryptor()
|
||||
encrypted = encryptor.update(BlockCipher.pad(text.encode())) + encryptor.finalize()
|
||||
return b64encode(iv + encrypted).decode()
|
||||
|
||||
@staticmethod
|
||||
def decrypt(key, ciphertext, leave_empty=True):
|
||||
"""Decrypt a message.
|
||||
|
||||
Based on http://stackoverflow.com/a/12525165
|
||||
|
||||
@param key (unicode): the decryption key
|
||||
@param ciphertext (base-64 encoded str): the text to decrypt
|
||||
@param leave_empty (bool): if True, empty ciphertext will be returned "as is"
|
||||
@return: Deferred: str or None if the password could not be decrypted
|
||||
"""
|
||||
if leave_empty and ciphertext == "":
|
||||
return ""
|
||||
ciphertext = b64decode(ciphertext)
|
||||
iv, ciphertext = (
|
||||
ciphertext[: BlockCipher.IV_SIZE],
|
||||
ciphertext[BlockCipher.IV_SIZE :],
|
||||
)
|
||||
key = key.encode()
|
||||
key = (
|
||||
key[: BlockCipher.MAX_KEY_SIZE]
|
||||
if len(key) >= BlockCipher.MAX_KEY_SIZE
|
||||
else BlockCipher.pad(key)
|
||||
)
|
||||
|
||||
cipher = Cipher(algorithms.AES(key), modes.CFB8(iv), backend=crypto_backend)
|
||||
decryptor = cipher.decryptor()
|
||||
decrypted = decryptor.update(ciphertext) + decryptor.finalize()
|
||||
return BlockCipher.unpad(decrypted)
|
||||
|
||||
@staticmethod
|
||||
def get_random_key(size=None, base64=False):
|
||||
"""Return a random key suitable for block cipher encryption.
|
||||
|
||||
Note: a good value for the key length is to make it as long as the block size.
|
||||
|
||||
@param size: key length in bytes, positive or null (default: BlockCipher.IV_SIZE)
|
||||
@param base64: if True, encode the result to base-64
|
||||
@return: str (eventually base-64 encoded)
|
||||
"""
|
||||
if size is None or size < 0:
|
||||
size = BlockCipher.IV_SIZE
|
||||
key = urandom(size)
|
||||
return b64encode(key) if base64 else key
|
||||
|
||||
@staticmethod
|
||||
def pad(s):
|
||||
"""Method from http://stackoverflow.com/a/12525165"""
|
||||
bs = BlockCipher.BLOCK_SIZE
|
||||
return s + (bs - len(s) % bs) * (chr(bs - len(s) % bs)).encode()
|
||||
|
||||
@staticmethod
|
||||
def unpad(s):
|
||||
"""Method from http://stackoverflow.com/a/12525165"""
|
||||
s = s.decode()
|
||||
return s[0 : -ord(s[-1])]
|
||||
|
||||
|
||||
class PasswordHasher:
|
||||
|
||||
SALT_LEN = 16 # 128 bits
|
||||
|
||||
@staticmethod
|
||||
def hash(password, salt=None, leave_empty=True):
|
||||
"""Hash a password.
|
||||
|
||||
@param password (str): the password to hash
|
||||
@param salt (base-64 encoded str): if not None, use the given salt instead of a random value
|
||||
@param leave_empty (bool): if True, empty password will be returned "as is"
|
||||
@return: Deferred: base-64 encoded str
|
||||
"""
|
||||
if leave_empty and password == "":
|
||||
return ""
|
||||
salt = (
|
||||
b64decode(salt)[: PasswordHasher.SALT_LEN]
|
||||
if salt
|
||||
else urandom(PasswordHasher.SALT_LEN)
|
||||
)
|
||||
|
||||
# we use PyCrypto's PBKDF2 arguments while porting to crytography, to stay
|
||||
# compatible with existing installations. But this is temporary and we need
|
||||
# to update them to more secure values.
|
||||
kdf = PBKDF2HMAC(
|
||||
# FIXME: SHA1() is not secure, it is used here for historical reasons
|
||||
# and must be changed as soon as possible
|
||||
algorithm=hashes.SHA1(),
|
||||
length=16,
|
||||
salt=salt,
|
||||
iterations=1000,
|
||||
backend=crypto_backend
|
||||
)
|
||||
key = kdf.derive(password.encode())
|
||||
return b64encode(salt + key).decode()
|
||||
|
||||
@staticmethod
|
||||
def verify(attempt, pwd_hash):
|
||||
"""Verify a password attempt.
|
||||
|
||||
@param attempt (str): the attempt to check
|
||||
@param pwd_hash (str): the hash of the password
|
||||
@return: Deferred: boolean
|
||||
"""
|
||||
assert isinstance(attempt, str)
|
||||
assert isinstance(pwd_hash, str)
|
||||
leave_empty = pwd_hash == ""
|
||||
attempt_hash = PasswordHasher.hash(attempt, pwd_hash, leave_empty)
|
||||
assert isinstance(attempt_hash, str)
|
||||
return attempt_hash == pwd_hash
|
||||
499
libervia/backend/memory/disco.py
Executable file
499
libervia/backend/memory/disco.py
Executable file
|
|
@ -0,0 +1,499 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
|
||||
# SAT: a jabber client
|
||||
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from typing import Optional
|
||||
from libervia.backend.core.i18n import _
|
||||
from libervia.backend.core import exceptions
|
||||
from libervia.backend.core.log import getLogger
|
||||
from libervia.backend.core.core_types import SatXMPPEntity
|
||||
|
||||
from twisted.words.protocols.jabber import jid
|
||||
from twisted.words.protocols.jabber.error import StanzaError
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import reactor
|
||||
from twisted.python import failure
|
||||
from libervia.backend.core.constants import Const as C
|
||||
from libervia.backend.tools import xml_tools
|
||||
from libervia.backend.memory import persistent
|
||||
from wokkel import disco
|
||||
from base64 import b64encode
|
||||
from hashlib import sha1
|
||||
|
||||
|
||||
log = getLogger(__name__)
|
||||
|
||||
|
||||
TIMEOUT = 15
|
||||
CAP_HASH_ERROR = "ERROR"
|
||||
|
||||
|
||||
class HashGenerationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ByteIdentity(object):
|
||||
"""This class manage identity as bytes (needed for i;octet sort), it is used for the hash generation"""
|
||||
|
||||
def __init__(self, identity, lang=None):
|
||||
assert isinstance(identity, disco.DiscoIdentity)
|
||||
self.category = identity.category.encode("utf-8")
|
||||
self.idType = identity.type.encode("utf-8")
|
||||
self.name = identity.name.encode("utf-8") if identity.name else b""
|
||||
self.lang = lang.encode("utf-8") if lang is not None else b""
|
||||
|
||||
def __bytes__(self):
|
||||
return b"%s/%s/%s/%s" % (self.category, self.idType, self.lang, self.name)
|
||||
|
||||
|
||||
class HashManager(object):
|
||||
"""map object which manage hashes
|
||||
|
||||
persistent storage is update when a new hash is added
|
||||
"""
|
||||
|
||||
def __init__(self, persistent):
|
||||
self.hashes = {
|
||||
CAP_HASH_ERROR: disco.DiscoInfo() # used when we can't get disco infos
|
||||
}
|
||||
self.persistent = persistent
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.hashes[key]
|
||||
|
||||
def __setitem__(self, hash_, disco_info):
|
||||
if hash_ in self.hashes:
|
||||
log.debug("ignoring hash set: it is already known")
|
||||
return
|
||||
self.hashes[hash_] = disco_info
|
||||
self.persistent[hash_] = disco_info.toElement().toXml()
|
||||
|
||||
def __contains__(self, hash_):
|
||||
return self.hashes.__contains__(hash_)
|
||||
|
||||
def load(self):
|
||||
def fill_hashes(hashes):
|
||||
for hash_, xml in hashes.items():
|
||||
element = xml_tools.ElementParser()(xml)
|
||||
disco_info = disco.DiscoInfo.fromElement(element)
|
||||
for ext_form in disco_info.extensions.values():
|
||||
# wokkel doesn't call typeCheck on reception, so we do it here
|
||||
ext_form.typeCheck()
|
||||
if not disco_info.features and not disco_info.identities:
|
||||
log.warning(
|
||||
_(
|
||||
"no feature/identity found in disco element (hash: {cap_hash}), ignoring: {xml}"
|
||||
).format(cap_hash=hash_, xml=xml)
|
||||
)
|
||||
else:
|
||||
self.hashes[hash_] = disco_info
|
||||
|
||||
log.info("Disco hashes loaded")
|
||||
|
||||
d = self.persistent.load()
|
||||
d.addCallback(fill_hashes)
|
||||
return d
|
||||
|
||||
|
||||
class Discovery(object):
|
||||
""" Manage capabilities of entities """
|
||||
|
||||
def __init__(self, host):
|
||||
self.host = host
|
||||
# TODO: remove legacy hashes
|
||||
|
||||
def load(self):
|
||||
"""Load persistent hashes"""
|
||||
self.hashes = HashManager(persistent.PersistentDict("disco"))
|
||||
return self.hashes.load()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def hasFeature(self, client, feature, jid_=None, node=""):
|
||||
"""Tell if an entity has the required feature
|
||||
|
||||
@param feature: feature namespace
|
||||
@param jid_: jid of the target, or None for profile's server
|
||||
@param node(unicode): optional node to use for disco request
|
||||
@return: a Deferred which fire a boolean (True if feature is available)
|
||||
"""
|
||||
disco_infos = yield self.get_infos(client, jid_, node)
|
||||
defer.returnValue(feature in disco_infos.features)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def check_feature(self, client, feature, jid_=None, node=""):
|
||||
"""Like hasFeature, but raise an exception is feature is not Found
|
||||
|
||||
@param feature: feature namespace
|
||||
@param jid_: jid of the target, or None for profile's server
|
||||
@param node(unicode): optional node to use for disco request
|
||||
|
||||
@raise: exceptions.FeatureNotFound
|
||||
"""
|
||||
disco_infos = yield self.get_infos(client, jid_, node)
|
||||
if not feature in disco_infos.features:
|
||||
raise failure.Failure(exceptions.FeatureNotFound())
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def check_features(self, client, features, jid_=None, identity=None, node=""):
|
||||
"""Like check_feature, but check several features at once, and check also identity
|
||||
|
||||
@param features(iterable[unicode]): features to check
|
||||
@param jid_(jid.JID): jid of the target, or None for profile's server
|
||||
@param node(unicode): optional node to use for disco request
|
||||
@param identity(None, tuple(unicode, unicode): if not None, the entity must have an identity with this (category, type) tuple
|
||||
|
||||
@raise: exceptions.FeatureNotFound
|
||||
"""
|
||||
disco_infos = yield self.get_infos(client, jid_, node)
|
||||
if not set(features).issubset(disco_infos.features):
|
||||
raise failure.Failure(exceptions.FeatureNotFound())
|
||||
|
||||
if identity is not None and identity not in disco_infos.identities:
|
||||
raise failure.Failure(exceptions.FeatureNotFound())
|
||||
|
||||
async def has_identity(
|
||||
self,
|
||||
client: SatXMPPEntity,
|
||||
category: str,
|
||||
type_: str,
|
||||
jid_: Optional[jid.JID] = None,
|
||||
node: str = ""
|
||||
) -> bool:
|
||||
"""Tell if an entity has the requested identity
|
||||
|
||||
@param category: identity category
|
||||
@param type_: identity type
|
||||
@param jid_: jid of the target, or None for profile's server
|
||||
@param node(unicode): optional node to use for disco request
|
||||
@return: True if the entity has the given identity
|
||||
"""
|
||||
disco_infos = await self.get_infos(client, jid_, node)
|
||||
return (category, type_) in disco_infos.identities
|
||||
|
||||
def get_infos(self, client, jid_=None, node="", use_cache=True):
|
||||
"""get disco infos from jid_, filling capability hash if needed
|
||||
|
||||
@param jid_: jid of the target, or None for profile's server
|
||||
@param node(unicode): optional node to use for disco request
|
||||
@param use_cache(bool): if True, use cached data if available
|
||||
@return: a Deferred which fire disco.DiscoInfo
|
||||
"""
|
||||
if jid_ is None:
|
||||
jid_ = jid.JID(client.jid.host)
|
||||
try:
|
||||
if not use_cache:
|
||||
# we ignore cache, so we pretend we haven't found it
|
||||
raise KeyError
|
||||
cap_hash = self.host.memory.entity_data_get(
|
||||
client, jid_, [C.ENTITY_CAP_HASH]
|
||||
)[C.ENTITY_CAP_HASH]
|
||||
except (KeyError, exceptions.UnknownEntityError):
|
||||
# capability hash is not available, we'll compute one
|
||||
def infos_cb(disco_infos):
|
||||
cap_hash = self.generate_hash(disco_infos)
|
||||
for ext_form in disco_infos.extensions.values():
|
||||
# wokkel doesn't call typeCheck on reception, so we do it here
|
||||
# to avoid ending up with incorrect types. We have to do it after
|
||||
# the hash has been generated (str value is needed to compute the
|
||||
# hash)
|
||||
ext_form.typeCheck()
|
||||
self.hashes[cap_hash] = disco_infos
|
||||
self.host.memory.update_entity_data(
|
||||
client, jid_, C.ENTITY_CAP_HASH, cap_hash
|
||||
)
|
||||
return disco_infos
|
||||
|
||||
def infos_eb(fail):
|
||||
if fail.check(defer.CancelledError):
|
||||
reason = "request time-out"
|
||||
fail = failure.Failure(exceptions.TimeOutError(str(fail.value)))
|
||||
else:
|
||||
try:
|
||||
reason = str(fail.value)
|
||||
except AttributeError:
|
||||
reason = str(fail)
|
||||
|
||||
log.warning(
|
||||
"can't request disco infos from {jid}: {reason}".format(
|
||||
jid=jid_.full(), reason=reason
|
||||
)
|
||||
)
|
||||
|
||||
# XXX we set empty disco in cache, to avoid getting an error or waiting
|
||||
# for a timeout again the next time
|
||||
self.host.memory.update_entity_data(
|
||||
client, jid_, C.ENTITY_CAP_HASH, CAP_HASH_ERROR
|
||||
)
|
||||
raise fail
|
||||
|
||||
d = client.disco.requestInfo(jid_, nodeIdentifier=node)
|
||||
d.addCallback(infos_cb)
|
||||
d.addErrback(infos_eb)
|
||||
return d
|
||||
else:
|
||||
disco_infos = self.hashes[cap_hash]
|
||||
return defer.succeed(disco_infos)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_items(self, client, jid_=None, node="", use_cache=True):
|
||||
"""get disco items from jid_, cache them for our own server
|
||||
|
||||
@param jid_(jid.JID): jid of the target, or None for profile's server
|
||||
@param node(unicode): optional node to use for disco request
|
||||
@param use_cache(bool): if True, use cached data if available
|
||||
@return: a Deferred which fire disco.DiscoItems
|
||||
"""
|
||||
if jid_ is None:
|
||||
jid_ = client.server_jid
|
||||
|
||||
if jid_ == client.server_jid and not node:
|
||||
# we cache items only for our own server and if node is not set
|
||||
try:
|
||||
items = self.host.memory.entity_data_get(
|
||||
client, jid_, ["DISCO_ITEMS"]
|
||||
)["DISCO_ITEMS"]
|
||||
log.debug("[%s] disco items are in cache" % jid_.full())
|
||||
if not use_cache:
|
||||
# we ignore cache, so we pretend we haven't found it
|
||||
raise KeyError
|
||||
except (KeyError, exceptions.UnknownEntityError):
|
||||
log.debug("Caching [%s] disco items" % jid_.full())
|
||||
items = yield client.disco.requestItems(jid_, nodeIdentifier=node)
|
||||
self.host.memory.update_entity_data(
|
||||
client, jid_, "DISCO_ITEMS", items
|
||||
)
|
||||
else:
|
||||
try:
|
||||
items = yield client.disco.requestItems(jid_, nodeIdentifier=node)
|
||||
except StanzaError as e:
|
||||
log.warning(
|
||||
"Error while requesting items for {jid}: {reason}".format(
|
||||
jid=jid_.full(), reason=e.condition
|
||||
)
|
||||
)
|
||||
items = disco.DiscoItems()
|
||||
|
||||
defer.returnValue(items)
|
||||
|
||||
def _infos_eb(self, failure_, entity_jid):
|
||||
failure_.trap(StanzaError)
|
||||
log.warning(
|
||||
_("Error while requesting [%(jid)s]: %(error)s")
|
||||
% {"jid": entity_jid.full(), "error": failure_.getErrorMessage()}
|
||||
)
|
||||
|
||||
def find_service_entity(self, client, category, type_, jid_=None):
|
||||
"""Helper method to find first available entity from find_service_entities
|
||||
|
||||
args are the same as for [find_service_entities]
|
||||
@return (jid.JID, None): found entity
|
||||
"""
|
||||
d = self.host.find_service_entities(client, category, type_)
|
||||
d.addCallback(lambda entities: entities.pop() if entities else None)
|
||||
return d
|
||||
|
||||
def find_service_entities(self, client, category, type_, jid_=None):
|
||||
"""Return all available items of an entity which correspond to (category, type_)
|
||||
|
||||
@param category: identity's category
|
||||
@param type_: identitiy's type
|
||||
@param jid_: the jid of the target server (None for profile's server)
|
||||
@return: a set of found entities
|
||||
@raise defer.CancelledError: the request timed out
|
||||
"""
|
||||
found_entities = set()
|
||||
|
||||
def infos_cb(infos, entity_jid):
|
||||
if (category, type_) in infos.identities:
|
||||
found_entities.add(entity_jid)
|
||||
|
||||
def got_items(items):
|
||||
defers_list = []
|
||||
for item in items:
|
||||
info_d = self.get_infos(client, item.entity)
|
||||
info_d.addCallbacks(
|
||||
infos_cb, self._infos_eb, [item.entity], None, [item.entity]
|
||||
)
|
||||
defers_list.append(info_d)
|
||||
return defer.DeferredList(defers_list)
|
||||
|
||||
d = self.get_items(client, jid_)
|
||||
d.addCallback(got_items)
|
||||
d.addCallback(lambda __: found_entities)
|
||||
reactor.callLater(
|
||||
TIMEOUT, d.cancel
|
||||
) # FIXME: one bad service make a general timeout
|
||||
return d
|
||||
|
||||
def find_features_set(self, client, features, identity=None, jid_=None):
|
||||
"""Return entities (including jid_ and its items) offering features
|
||||
|
||||
@param features: iterable of features which must be present
|
||||
@param identity(None, tuple(unicode, unicode)): if not None, accept only this
|
||||
(category/type) identity
|
||||
@param jid_: the jid of the target server (None for profile's server)
|
||||
@param profile: %(doc_profile)s
|
||||
@return: a set of found entities
|
||||
"""
|
||||
if jid_ is None:
|
||||
jid_ = jid.JID(client.jid.host)
|
||||
features = set(features)
|
||||
found_entities = set()
|
||||
|
||||
def infos_cb(infos, entity):
|
||||
if entity is None:
|
||||
log.warning(_("received an item without jid"))
|
||||
return
|
||||
if identity is not None and identity not in infos.identities:
|
||||
return
|
||||
if features.issubset(infos.features):
|
||||
found_entities.add(entity)
|
||||
|
||||
def got_items(items):
|
||||
defer_list = []
|
||||
for entity in [jid_] + [item.entity for item in items]:
|
||||
infos_d = self.get_infos(client, entity)
|
||||
infos_d.addCallbacks(infos_cb, self._infos_eb, [entity], None, [entity])
|
||||
defer_list.append(infos_d)
|
||||
return defer.DeferredList(defer_list)
|
||||
|
||||
d = self.get_items(client, jid_)
|
||||
d.addCallback(got_items)
|
||||
d.addCallback(lambda __: found_entities)
|
||||
reactor.callLater(
|
||||
TIMEOUT, d.cancel
|
||||
) # FIXME: one bad service make a general timeout
|
||||
return d
|
||||
|
||||
def generate_hash(self, services):
|
||||
""" Generate a unique hash for given service
|
||||
|
||||
hash algorithm is the one described in XEP-0115
|
||||
@param services: iterable of disco.DiscoIdentity/disco.DiscoFeature, as returned by discoHandler.info
|
||||
|
||||
"""
|
||||
s = []
|
||||
# identities
|
||||
byte_identities = [
|
||||
ByteIdentity(service)
|
||||
for service in services
|
||||
if isinstance(service, disco.DiscoIdentity)
|
||||
] # FIXME: lang must be managed here
|
||||
byte_identities.sort(key=lambda i: i.lang)
|
||||
byte_identities.sort(key=lambda i: i.idType)
|
||||
byte_identities.sort(key=lambda i: i.category)
|
||||
for identity in byte_identities:
|
||||
s.append(bytes(identity))
|
||||
s.append(b"<")
|
||||
# features
|
||||
byte_features = [
|
||||
service.encode("utf-8")
|
||||
for service in services
|
||||
if isinstance(service, disco.DiscoFeature)
|
||||
]
|
||||
byte_features.sort() # XXX: the default sort has the same behaviour as the requested RFC 4790 i;octet sort
|
||||
for feature in byte_features:
|
||||
s.append(feature)
|
||||
s.append(b"<")
|
||||
|
||||
# extensions
|
||||
ext = list(services.extensions.values())
|
||||
ext.sort(key=lambda f: f.formNamespace.encode('utf-8'))
|
||||
for extension in ext:
|
||||
s.append(extension.formNamespace.encode('utf-8'))
|
||||
s.append(b"<")
|
||||
fields = extension.fieldList
|
||||
fields.sort(key=lambda f: f.var.encode('utf-8'))
|
||||
for field in fields:
|
||||
s.append(field.var.encode('utf-8'))
|
||||
s.append(b"<")
|
||||
values = [v.encode('utf-8') for v in field.values]
|
||||
values.sort()
|
||||
for value in values:
|
||||
s.append(value)
|
||||
s.append(b"<")
|
||||
|
||||
cap_hash = b64encode(sha1(b"".join(s)).digest()).decode('utf-8')
|
||||
log.debug(_("Capability hash generated: [{cap_hash}]").format(cap_hash=cap_hash))
|
||||
return cap_hash
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _disco_infos(
|
||||
self, entity_jid_s, node="", use_cache=True, profile_key=C.PROF_KEY_NONE
|
||||
):
|
||||
"""Discovery method for the bridge
|
||||
@param entity_jid_s: entity we want to discover
|
||||
@param use_cache(bool): if True, use cached data if available
|
||||
@param node(unicode): optional node to use
|
||||
|
||||
@return: list of tuples
|
||||
"""
|
||||
client = self.host.get_client(profile_key)
|
||||
entity = jid.JID(entity_jid_s)
|
||||
disco_infos = yield self.get_infos(client, entity, node, use_cache)
|
||||
extensions = {}
|
||||
# FIXME: should extensions be serialised using tools.common.data_format?
|
||||
for form_type, form in list(disco_infos.extensions.items()):
|
||||
fields = []
|
||||
for field in form.fieldList:
|
||||
data = {"type": field.fieldType}
|
||||
for attr in ("var", "label", "desc"):
|
||||
value = getattr(field, attr)
|
||||
if value is not None:
|
||||
data[attr] = value
|
||||
|
||||
values = [field.value] if field.value is not None else field.values
|
||||
if field.fieldType == "boolean":
|
||||
values = [C.bool_const(v) for v in values]
|
||||
fields.append((data, values))
|
||||
|
||||
extensions[form_type or ""] = fields
|
||||
|
||||
defer.returnValue((
|
||||
[str(f) for f in disco_infos.features],
|
||||
[(cat, type_, name or "")
|
||||
for (cat, type_), name in list(disco_infos.identities.items())],
|
||||
extensions))
|
||||
|
||||
def items2tuples(self, disco_items):
|
||||
"""convert disco items to tuple of strings
|
||||
|
||||
@param disco_items(iterable[disco.DiscoItem]): items
|
||||
@return G(tuple[unicode,unicode,unicode]): serialised items
|
||||
"""
|
||||
for item in disco_items:
|
||||
if not item.entity:
|
||||
log.warning(_("invalid item (no jid)"))
|
||||
continue
|
||||
yield (item.entity.full(), item.nodeIdentifier or "", item.name or "")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _disco_items(
|
||||
self, entity_jid_s, node="", use_cache=True, profile_key=C.PROF_KEY_NONE
|
||||
):
|
||||
""" Discovery method for the bridge
|
||||
|
||||
@param entity_jid_s: entity we want to discover
|
||||
@param node(unicode): optional node to use
|
||||
@param use_cache(bool): if True, use cached data if available
|
||||
@return: list of tuples"""
|
||||
client = self.host.get_client(profile_key)
|
||||
entity = jid.JID(entity_jid_s)
|
||||
disco_items = yield self.get_items(client, entity, node, use_cache)
|
||||
ret = list(self.items2tuples(disco_items))
|
||||
defer.returnValue(ret)
|
||||
534
libervia/backend/memory/encryption.py
Normal file
534
libervia/backend/memory/encryption.py
Normal file
|
|
@ -0,0 +1,534 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
|
||||
# SAT: a jabber client
|
||||
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import copy
|
||||
from functools import partial
|
||||
from typing import Optional
|
||||
from twisted.words.protocols.jabber import jid
|
||||
from twisted.internet import defer
|
||||
from twisted.python import failure
|
||||
from libervia.backend.core.core_types import EncryptionPlugin, EncryptionSession, MessageData
|
||||
from libervia.backend.core.i18n import D_, _
|
||||
from libervia.backend.core.constants import Const as C
|
||||
from libervia.backend.core import exceptions
|
||||
from libervia.backend.core.log import getLogger
|
||||
from libervia.backend.tools.common import data_format
|
||||
from libervia.backend.tools import utils
|
||||
from libervia.backend.memory import persistent
|
||||
|
||||
|
||||
log = getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptionHandler:
|
||||
"""Class to handle encryption sessions for a client"""
|
||||
plugins = [] # plugin able to encrypt messages
|
||||
|
||||
def __init__(self, client):
|
||||
self.client = client
|
||||
self._sessions = {} # bare_jid ==> encryption_data
|
||||
self._stored_session = persistent.PersistentDict(
|
||||
"core:encryption", profile=client.profile)
|
||||
|
||||
@property
|
||||
def host(self):
|
||||
return self.client.host_app
|
||||
|
||||
async def load_sessions(self):
|
||||
"""Load persistent sessions"""
|
||||
await self._stored_session.load()
|
||||
start_d_list = []
|
||||
for entity_jid_s, namespace in self._stored_session.items():
|
||||
entity = jid.JID(entity_jid_s)
|
||||
start_d_list.append(defer.ensureDeferred(self.start(entity, namespace)))
|
||||
|
||||
if start_d_list:
|
||||
result = await defer.DeferredList(start_d_list)
|
||||
for idx, (success, err) in enumerate(result):
|
||||
if not success:
|
||||
entity_jid_s, namespace = list(self._stored_session.items())[idx]
|
||||
log.warning(_(
|
||||
"Could not restart {namespace!r} encryption with {entity}: {err}"
|
||||
).format(namespace=namespace, entity=entity_jid_s, err=err))
|
||||
log.info(_("encryption sessions restored"))
|
||||
|
||||
@classmethod
|
||||
def register_plugin(cls, plg_instance, name, namespace, priority=0, directed=False):
|
||||
"""Register a plugin handling an encryption algorithm
|
||||
|
||||
@param plg_instance(object): instance of the plugin
|
||||
it must have the following methods:
|
||||
- get_trust_ui(entity): return a XMLUI for trust management
|
||||
entity(jid.JID): entity to manage
|
||||
The returned XMLUI must be a form
|
||||
if may have the following methods:
|
||||
- start_encryption(entity): start encrypted session
|
||||
entity(jid.JID): entity to start encrypted session with
|
||||
- stop_encryption(entity): start encrypted session
|
||||
entity(jid.JID): entity to stop encrypted session with
|
||||
if they don't exists, those 2 methods will be ignored.
|
||||
|
||||
@param name(unicode): human readable name of the encryption algorithm
|
||||
@param namespace(unicode): namespace of the encryption algorithm
|
||||
@param priority(int): priority of this plugin to encrypt an message when not
|
||||
selected manually
|
||||
@param directed(bool): True if this plugin is directed (if it works with one
|
||||
device only at a time)
|
||||
"""
|
||||
existing_ns = set()
|
||||
existing_names = set()
|
||||
for p in cls.plugins:
|
||||
existing_ns.add(p.namespace.lower())
|
||||
existing_names.add(p.name.lower())
|
||||
if namespace.lower() in existing_ns:
|
||||
raise exceptions.ConflictError("A plugin with this namespace already exists!")
|
||||
if name.lower() in existing_names:
|
||||
raise exceptions.ConflictError("A plugin with this name already exists!")
|
||||
plugin = EncryptionPlugin(
|
||||
instance=plg_instance,
|
||||
name=name,
|
||||
namespace=namespace,
|
||||
priority=priority,
|
||||
directed=directed)
|
||||
cls.plugins.append(plugin)
|
||||
cls.plugins.sort(key=lambda p: p.priority)
|
||||
log.info(_("Encryption plugin registered: {name}").format(name=name))
|
||||
|
||||
@classmethod
|
||||
def getPlugins(cls):
|
||||
return cls.plugins
|
||||
|
||||
@classmethod
|
||||
def get_plugin(cls, namespace):
|
||||
try:
|
||||
return next(p for p in cls.plugins if p.namespace == namespace)
|
||||
except StopIteration:
|
||||
raise exceptions.NotFound(_(
|
||||
"Can't find requested encryption plugin: {namespace}").format(
|
||||
namespace=namespace))
|
||||
|
||||
@classmethod
|
||||
def get_namespaces(cls):
|
||||
"""Get available plugin namespaces"""
|
||||
return {p.namespace for p in cls.getPlugins()}
|
||||
|
||||
@classmethod
|
||||
def get_ns_from_name(cls, name):
|
||||
"""Retrieve plugin namespace from its name
|
||||
|
||||
@param name(unicode): name of the plugin (case insensitive)
|
||||
@return (unicode): namespace of the plugin
|
||||
@raise exceptions.NotFound: there is not encryption plugin of this name
|
||||
"""
|
||||
for p in cls.plugins:
|
||||
if p.name.lower() == name.lower():
|
||||
return p.namespace
|
||||
raise exceptions.NotFound(_(
|
||||
"Can't find a plugin with the name \"{name}\".".format(
|
||||
name=name)))
|
||||
|
||||
def get_bridge_data(self, session):
|
||||
"""Retrieve session data serialized for bridge.
|
||||
|
||||
@param session(dict): encryption session
|
||||
@return (unicode): serialized data for bridge
|
||||
"""
|
||||
if session is None:
|
||||
return ''
|
||||
plugin = session['plugin']
|
||||
bridge_data = {'name': plugin.name,
|
||||
'namespace': plugin.namespace}
|
||||
if 'directed_devices' in session:
|
||||
bridge_data['directed_devices'] = session['directed_devices']
|
||||
|
||||
return data_format.serialise(bridge_data)
|
||||
|
||||
async def _start_encryption(self, plugin, entity):
|
||||
"""Start encryption with a plugin
|
||||
|
||||
This method must be called just before adding a plugin session.
|
||||
StartEncryptionn method of plugin will be called if it exists.
|
||||
"""
|
||||
if not plugin.directed:
|
||||
await self._stored_session.aset(entity.userhost(), plugin.namespace)
|
||||
try:
|
||||
start_encryption = plugin.instance.start_encryption
|
||||
except AttributeError:
|
||||
log.debug(f"No start_encryption method found for {plugin.namespace}")
|
||||
else:
|
||||
# we copy entity to avoid having the resource changed by stop_encryption
|
||||
await utils.as_deferred(start_encryption, self.client, copy.copy(entity))
|
||||
|
||||
async def _stop_encryption(self, plugin, entity):
|
||||
"""Stop encryption with a plugin
|
||||
|
||||
This method must be called just before removing a plugin session.
|
||||
StopEncryptionn method of plugin will be called if it exists.
|
||||
"""
|
||||
try:
|
||||
await self._stored_session.adel(entity.userhost())
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
stop_encryption = plugin.instance.stop_encryption
|
||||
except AttributeError:
|
||||
log.debug(f"No stop_encryption method found for {plugin.namespace}")
|
||||
else:
|
||||
# we copy entity to avoid having the resource changed by stop_encryption
|
||||
return utils.as_deferred(stop_encryption, self.client, copy.copy(entity))
|
||||
|
||||
async def start(self, entity, namespace=None, replace=False):
|
||||
"""Start an encryption session with an entity
|
||||
|
||||
@param entity(jid.JID): entity to start an encryption session with
|
||||
must be bare jid is the algorithm encrypt for all devices
|
||||
@param namespace(unicode, None): namespace of the encryption algorithm
|
||||
to use.
|
||||
None to select automatically an algorithm
|
||||
@param replace(bool): if True and an encrypted session already exists,
|
||||
it will be replaced by the new one
|
||||
"""
|
||||
if not self.plugins:
|
||||
raise exceptions.NotFound(_("No encryption plugin is registered, "
|
||||
"an encryption session can't be started"))
|
||||
|
||||
if namespace is None:
|
||||
plugin = self.plugins[0]
|
||||
else:
|
||||
plugin = self.get_plugin(namespace)
|
||||
|
||||
bare_jid = entity.userhostJID()
|
||||
if bare_jid in self._sessions:
|
||||
# we have already an encryption session with this contact
|
||||
former_plugin = self._sessions[bare_jid]["plugin"]
|
||||
if former_plugin.namespace == namespace:
|
||||
log.info(_("Session with {bare_jid} is already encrypted with {name}. "
|
||||
"Nothing to do.").format(
|
||||
bare_jid=bare_jid, name=former_plugin.name))
|
||||
return
|
||||
|
||||
if replace:
|
||||
# there is a conflict, but replacement is requested
|
||||
# so we stop previous encryption to use new one
|
||||
del self._sessions[bare_jid]
|
||||
await self._stop_encryption(former_plugin, entity)
|
||||
else:
|
||||
msg = (_("Session with {bare_jid} is already encrypted with {name}. "
|
||||
"Please stop encryption session before changing algorithm.")
|
||||
.format(bare_jid=bare_jid, name=plugin.name))
|
||||
log.warning(msg)
|
||||
raise exceptions.ConflictError(msg)
|
||||
|
||||
data = {"plugin": plugin}
|
||||
if plugin.directed:
|
||||
if not entity.resource:
|
||||
entity.resource = self.host.memory.main_resource_get(self.client, entity)
|
||||
if not entity.resource:
|
||||
raise exceptions.NotFound(
|
||||
_("No resource found for {destinee}, can't encrypt with {name}")
|
||||
.format(destinee=entity.full(), name=plugin.name))
|
||||
log.info(_("No resource specified to encrypt with {name}, using "
|
||||
"{destinee}.").format(destinee=entity.full(),
|
||||
name=plugin.name))
|
||||
# indicate that we encrypt only for some devices
|
||||
directed_devices = data['directed_devices'] = [entity.resource]
|
||||
elif entity.resource:
|
||||
raise ValueError(_("{name} encryption must be used with bare jids."))
|
||||
|
||||
await self._start_encryption(plugin, entity)
|
||||
self._sessions[entity.userhostJID()] = data
|
||||
log.info(_("Encryption session has been set for {entity_jid} with "
|
||||
"{encryption_name}").format(
|
||||
entity_jid=entity.full(), encryption_name=plugin.name))
|
||||
self.host.bridge.message_encryption_started(
|
||||
entity.full(),
|
||||
self.get_bridge_data(data),
|
||||
self.client.profile)
|
||||
msg = D_("Encryption session started: your messages with {destinee} are "
|
||||
"now end to end encrypted using {name} algorithm.").format(
|
||||
destinee=entity.full(), name=plugin.name)
|
||||
directed_devices = data.get('directed_devices')
|
||||
if directed_devices:
|
||||
msg += "\n" + D_("Message are encrypted only for {nb_devices} device(s): "
|
||||
"{devices_list}.").format(
|
||||
nb_devices=len(directed_devices),
|
||||
devices_list = ', '.join(directed_devices))
|
||||
|
||||
self.client.feedback(bare_jid, msg)
|
||||
|
||||
async def stop(self, entity, namespace=None):
|
||||
"""Stop an encryption session with an entity
|
||||
|
||||
@param entity(jid.JID): entity with who the encryption session must be stopped
|
||||
must be bare jid if the algorithm encrypt for all devices
|
||||
@param namespace(unicode): namespace of the session to stop
|
||||
when specified, used to check that we stop the right encryption session
|
||||
"""
|
||||
session = self.getSession(entity.userhostJID())
|
||||
if not session:
|
||||
raise failure.Failure(
|
||||
exceptions.NotFound(_("There is no encryption session with this "
|
||||
"entity.")))
|
||||
plugin = session['plugin']
|
||||
if namespace is not None and plugin.namespace != namespace:
|
||||
raise exceptions.InternalError(_(
|
||||
"The encryption session is not run with the expected plugin: encrypted "
|
||||
"with {current_name} and was expecting {expected_name}").format(
|
||||
current_name=session['plugin'].namespace,
|
||||
expected_name=namespace))
|
||||
if entity.resource:
|
||||
try:
|
||||
directed_devices = session['directed_devices']
|
||||
except KeyError:
|
||||
raise exceptions.NotFound(_(
|
||||
"There is a session for the whole entity (i.e. all devices of the "
|
||||
"entity), not a directed one. Please use bare jid if you want to "
|
||||
"stop the whole encryption with this entity."))
|
||||
|
||||
try:
|
||||
directed_devices.remove(entity.resource)
|
||||
except ValueError:
|
||||
raise exceptions.NotFound(_("There is no directed session with this "
|
||||
"entity."))
|
||||
else:
|
||||
if not directed_devices:
|
||||
# if we have no more directed device sessions,
|
||||
# we stop the whole session
|
||||
# see comment below for deleting session before stopping encryption
|
||||
del self._sessions[entity.userhostJID()]
|
||||
await self._stop_encryption(plugin, entity)
|
||||
else:
|
||||
# plugin's stop_encryption may call stop again (that's the case with OTR)
|
||||
# so we need to remove plugin from session before calling self._stop_encryption
|
||||
del self._sessions[entity.userhostJID()]
|
||||
await self._stop_encryption(plugin, entity)
|
||||
|
||||
log.info(_("encryption session stopped with entity {entity}").format(
|
||||
entity=entity.full()))
|
||||
self.host.bridge.message_encryption_stopped(
|
||||
entity.full(),
|
||||
{'name': plugin.name,
|
||||
'namespace': plugin.namespace,
|
||||
},
|
||||
self.client.profile)
|
||||
msg = D_("Encryption session finished: your messages with {destinee} are "
|
||||
"NOT end to end encrypted anymore.\nYour server administrators or "
|
||||
"{destinee} server administrators will be able to read them.").format(
|
||||
destinee=entity.full())
|
||||
|
||||
self.client.feedback(entity, msg)
|
||||
|
||||
def getSession(self, entity: jid.JID) -> Optional[EncryptionSession]:
|
||||
"""Get encryption session for this contact
|
||||
|
||||
@param entity(jid.JID): get the session for this entity
|
||||
must be a bare jid
|
||||
@return (dict, None): encryption session data
|
||||
None if there is not encryption for this session with this jid
|
||||
"""
|
||||
if entity.resource:
|
||||
raise ValueError("Full jid given when expecting bare jid")
|
||||
return self._sessions.get(entity)
|
||||
|
||||
def get_namespace(self, entity: jid.JID) -> Optional[str]:
|
||||
"""Helper method to get the current encryption namespace used
|
||||
|
||||
@param entity: get the namespace for this entity must be a bare jid
|
||||
@return: the algorithm namespace currently used in this session, or None if no
|
||||
e2ee is currently used.
|
||||
"""
|
||||
session = self.getSession(entity)
|
||||
if session is None:
|
||||
return None
|
||||
return session["plugin"].namespace
|
||||
|
||||
def get_trust_ui(self, entity_jid, namespace=None):
|
||||
"""Retrieve encryption UI
|
||||
|
||||
@param entity_jid(jid.JID): get the UI for this entity
|
||||
must be a bare jid
|
||||
@param namespace(unicode): namespace of the algorithm to manage
|
||||
if None use current algorithm
|
||||
@return D(xmlui): XMLUI for trust management
|
||||
the xmlui is a form
|
||||
None if there is not encryption for this session with this jid
|
||||
@raise exceptions.NotFound: no algorithm/plugin found
|
||||
@raise NotImplementedError: plugin doesn't handle UI management
|
||||
"""
|
||||
if namespace is None:
|
||||
session = self.getSession(entity_jid)
|
||||
if not session:
|
||||
raise exceptions.NotFound(
|
||||
"No encryption session currently active for {entity_jid}"
|
||||
.format(entity_jid=entity_jid.full()))
|
||||
plugin = session['plugin']
|
||||
else:
|
||||
plugin = self.get_plugin(namespace)
|
||||
try:
|
||||
get_trust_ui = plugin.instance.get_trust_ui
|
||||
except AttributeError:
|
||||
raise NotImplementedError(
|
||||
"Encryption plugin doesn't handle trust management UI")
|
||||
else:
|
||||
return utils.as_deferred(get_trust_ui, self.client, entity_jid)
|
||||
|
||||
## Menus ##
|
||||
|
||||
@classmethod
|
||||
def _import_menus(cls, host):
|
||||
host.import_menu(
|
||||
(D_("Encryption"), D_("unencrypted (plain text)")),
|
||||
partial(cls._on_menu_unencrypted, host=host),
|
||||
security_limit=0,
|
||||
help_string=D_("End encrypted session"),
|
||||
type_=C.MENU_SINGLE,
|
||||
)
|
||||
for plg in cls.getPlugins():
|
||||
host.import_menu(
|
||||
(D_("Encryption"), plg.name),
|
||||
partial(cls._on_menu_name, host=host, plg=plg),
|
||||
security_limit=0,
|
||||
help_string=D_("Start {name} session").format(name=plg.name),
|
||||
type_=C.MENU_SINGLE,
|
||||
)
|
||||
host.import_menu(
|
||||
(D_("Encryption"), D_("⛨ {name} trust").format(name=plg.name)),
|
||||
partial(cls._on_menu_trust, host=host, plg=plg),
|
||||
security_limit=0,
|
||||
help_string=D_("Manage {name} trust").format(name=plg.name),
|
||||
type_=C.MENU_SINGLE,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _on_menu_unencrypted(cls, data, host, profile):
|
||||
client = host.get_client(profile)
|
||||
peer_jid = jid.JID(data['jid']).userhostJID()
|
||||
d = defer.ensureDeferred(client.encryption.stop(peer_jid))
|
||||
d.addCallback(lambda __: {})
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def _on_menu_name(cls, data, host, plg, profile):
|
||||
client = host.get_client(profile)
|
||||
peer_jid = jid.JID(data['jid'])
|
||||
if not plg.directed:
|
||||
peer_jid = peer_jid.userhostJID()
|
||||
d = defer.ensureDeferred(
|
||||
client.encryption.start(peer_jid, plg.namespace, replace=True))
|
||||
d.addCallback(lambda __: {})
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
@defer.inlineCallbacks
|
||||
def _on_menu_trust(cls, data, host, plg, profile):
|
||||
client = host.get_client(profile)
|
||||
peer_jid = jid.JID(data['jid']).userhostJID()
|
||||
ui = yield client.encryption.get_trust_ui(peer_jid, plg.namespace)
|
||||
defer.returnValue({'xmlui': ui.toXml()})
|
||||
|
||||
## Triggers ##
|
||||
|
||||
def set_encryption_flag(self, mess_data):
|
||||
"""Set "encryption" key in mess_data if session with destinee is encrypted"""
|
||||
to_jid = mess_data['to']
|
||||
encryption = self._sessions.get(to_jid.userhostJID())
|
||||
if encryption is not None:
|
||||
plugin = encryption['plugin']
|
||||
if mess_data["type"] == "groupchat" and plugin.directed:
|
||||
raise exceptions.InternalError(
|
||||
f"encryption flag must not be set for groupchat if encryption algorithm "
|
||||
f"({encryption['plugin'].name}) is directed!")
|
||||
mess_data[C.MESS_KEY_ENCRYPTION] = encryption
|
||||
self.mark_as_encrypted(mess_data, plugin.namespace)
|
||||
|
||||
## Misc ##
|
||||
|
||||
def mark_as_encrypted(self, mess_data, namespace):
|
||||
"""Helper method to mark a message as having been e2e encrypted.
|
||||
|
||||
This should be used in the post_treat workflow of message_received trigger of
|
||||
the plugin
|
||||
@param mess_data(dict): message data as used in post treat workflow
|
||||
@param namespace(str): namespace of the algorithm used for encrypting the message
|
||||
"""
|
||||
mess_data['extra'][C.MESS_KEY_ENCRYPTED] = True
|
||||
from_bare_jid = mess_data['from'].userhostJID()
|
||||
if from_bare_jid != self.client.jid.userhostJID():
|
||||
session = self.getSession(from_bare_jid)
|
||||
if session is None:
|
||||
# if we are currently unencrypted, we start a session automatically
|
||||
# to avoid sending unencrypted messages in an encrypted context
|
||||
log.info(_(
|
||||
"Starting e2e session with {peer_jid} as we receive encrypted "
|
||||
"messages")
|
||||
.format(peer_jid=from_bare_jid)
|
||||
)
|
||||
defer.ensureDeferred(self.start(from_bare_jid, namespace))
|
||||
|
||||
return mess_data
|
||||
|
||||
def is_encryption_requested(
|
||||
self,
|
||||
mess_data: MessageData,
|
||||
namespace: Optional[str] = None
|
||||
) -> bool:
|
||||
"""Helper method to check if encryption is requested in an outgoind message
|
||||
|
||||
@param mess_data: message data for outgoing message
|
||||
@param namespace: if set, check if encryption is requested for the algorithm
|
||||
specified
|
||||
@return: True if the encryption flag is present
|
||||
"""
|
||||
encryption = mess_data.get(C.MESS_KEY_ENCRYPTION)
|
||||
if encryption is None:
|
||||
return False
|
||||
# we get plugin even if namespace is None to be sure that the key exists
|
||||
plugin = encryption['plugin']
|
||||
if namespace is None:
|
||||
return True
|
||||
return plugin.namespace == namespace
|
||||
|
||||
def isEncrypted(self, mess_data):
|
||||
"""Helper method to check if a message has the e2e encrypted flag
|
||||
|
||||
@param mess_data(dict): message data
|
||||
@return (bool): True if the encrypted flag is present
|
||||
"""
|
||||
return mess_data['extra'].get(C.MESS_KEY_ENCRYPTED, False)
|
||||
|
||||
|
||||
def mark_as_trusted(self, mess_data):
|
||||
"""Helper methor to mark a message as sent from a trusted entity.
|
||||
|
||||
This should be used in the post_treat workflow of message_received trigger of
|
||||
the plugin
|
||||
@param mess_data(dict): message data as used in post treat workflow
|
||||
"""
|
||||
mess_data[C.MESS_KEY_TRUSTED] = True
|
||||
return mess_data
|
||||
|
||||
def mark_as_untrusted(self, mess_data):
|
||||
"""Helper methor to mark a message as sent from an untrusted entity.
|
||||
|
||||
This should be used in the post_treat workflow of message_received trigger of
|
||||
the plugin
|
||||
@param mess_data(dict): message data as used in post treat workflow
|
||||
"""
|
||||
mess_data['trusted'] = False
|
||||
return mess_data
|
||||
1881
libervia/backend/memory/memory.py
Normal file
1881
libervia/backend/memory/memory.py
Normal file
File diff suppressed because it is too large
Load diff
3
libervia/backend/memory/migration/README
Normal file
3
libervia/backend/memory/migration/README
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
This directory and subdirectories contains Alembic migration scripts.
|
||||
|
||||
Please check Libervia documentation for details.
|
||||
0
libervia/backend/memory/migration/__init__.py
Normal file
0
libervia/backend/memory/migration/__init__.py
Normal file
89
libervia/backend/memory/migration/alembic.ini
Normal file
89
libervia/backend/memory/migration/alembic.ini
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts
|
||||
script_location = %(here)s
|
||||
|
||||
# template used to generate migration files
|
||||
# file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory.
|
||||
# prepend_sys_path = .
|
||||
|
||||
# timezone to use when rendering the date
|
||||
# within the migration file as well as the filename.
|
||||
# string value is passed to dateutil.tz.gettz()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the
|
||||
# "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; this defaults
|
||||
# to migration/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path
|
||||
# version_locations = %(here)s/bar %(here)s/bat migration/versions
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
# sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
93
libervia/backend/memory/migration/env.py
Normal file
93
libervia/backend/memory/migration/env.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import asyncio
|
||||
from logging.config import fileConfig
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from alembic import context
|
||||
from libervia.backend.memory import sqla_config
|
||||
from libervia.backend.memory.sqla_mapping import Base
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
# from myapp import mymodel
|
||||
# target_metadata = mymodel.Base.metadata
|
||||
target_metadata = Base.metadata
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline():
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
db_config = sqla_config.get_db_config()
|
||||
context.configure(
|
||||
url=db_config["url"],
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def include_name(name, type_, parent_names):
|
||||
if type_ == "table":
|
||||
if name.startswith("pubsub_items_fts"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def do_run_migrations(connection):
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=True,
|
||||
include_name=include_name
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def run_migrations_online():
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
db_config = sqla_config.get_db_config()
|
||||
engine = create_async_engine(
|
||||
db_config["url"],
|
||||
poolclass=pool.NullPool,
|
||||
future=True,
|
||||
)
|
||||
|
||||
async with engine.connect() as connection:
|
||||
await connection.run_sync(do_run_migrations)
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
asyncio.run(run_migrations_online())
|
||||
24
libervia/backend/memory/migration/script.py.mako
Normal file
24
libervia/backend/memory/migration/script.py.mako
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
branch_labels = ${repr(branch_labels)}
|
||||
depends_on = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade():
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade():
|
||||
${downgrades if downgrades else "pass"}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
"""create virtual table for Full-Text Search
|
||||
|
||||
Revision ID: 129ac51807e4
|
||||
Revises: 8974efc51d22
|
||||
Create Date: 2021-08-13 19:13:54.112538
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '129ac51807e4'
|
||||
down_revision = '8974efc51d22'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
queries = [
|
||||
"CREATE VIRTUAL TABLE pubsub_items_fts "
|
||||
"USING fts5(data, content=pubsub_items, content_rowid=id)",
|
||||
"CREATE TRIGGER pubsub_items_fts_sync_ins AFTER INSERT ON pubsub_items BEGIN"
|
||||
" INSERT INTO pubsub_items_fts(rowid, data) VALUES (new.id, new.data);"
|
||||
"END",
|
||||
"CREATE TRIGGER pubsub_items_fts_sync_del AFTER DELETE ON pubsub_items BEGIN"
|
||||
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) "
|
||||
"VALUES('delete', old.id, old.data);"
|
||||
"END",
|
||||
"CREATE TRIGGER pubsub_items_fts_sync_upd AFTER UPDATE ON pubsub_items BEGIN"
|
||||
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) VALUES"
|
||||
"('delete', old.id, old.data);"
|
||||
" INSERT INTO pubsub_items_fts(rowid, data) VALUES(new.id, new.data);"
|
||||
"END",
|
||||
"INSERT INTO pubsub_items_fts(rowid, data) SELECT id, data from pubsub_items"
|
||||
]
|
||||
for q in queries:
|
||||
op.execute(sa.DDL(q))
|
||||
|
||||
|
||||
def downgrade():
|
||||
queries = [
|
||||
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_ins",
|
||||
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_del",
|
||||
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_upd",
|
||||
"DROP TABLE IF EXISTS pubsub_items_fts",
|
||||
]
|
||||
for q in queries:
|
||||
op.execute(sa.DDL(q))
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
"""add origin_id column to history and adapt constraints
|
||||
|
||||
Revision ID: 4b002773cf92
|
||||
Revises: 79e5f3313fa4
|
||||
Create Date: 2022-06-13 16:10:39.711634
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '4b002773cf92'
|
||||
down_revision = '79e5f3313fa4'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
with op.batch_alter_table('history', schema=None) as batch_op:
|
||||
batch_op.add_column(sa.Column('origin_id', sa.Text(), nullable=True))
|
||||
batch_op.create_unique_constraint('uq_origin_id', ['profile_id', 'origin_id', 'source'])
|
||||
|
||||
with op.batch_alter_table('message', schema=None) as batch_op:
|
||||
batch_op.alter_column('history_uid',
|
||||
existing_type=sa.TEXT(),
|
||||
nullable=False)
|
||||
batch_op.alter_column('message',
|
||||
existing_type=sa.TEXT(),
|
||||
nullable=False)
|
||||
|
||||
with op.batch_alter_table('subject', schema=None) as batch_op:
|
||||
batch_op.alter_column('history_uid',
|
||||
existing_type=sa.TEXT(),
|
||||
nullable=False)
|
||||
batch_op.alter_column('subject',
|
||||
existing_type=sa.TEXT(),
|
||||
nullable=False)
|
||||
|
||||
|
||||
def downgrade():
|
||||
with op.batch_alter_table('subject', schema=None) as batch_op:
|
||||
batch_op.alter_column('subject',
|
||||
existing_type=sa.TEXT(),
|
||||
nullable=True)
|
||||
batch_op.alter_column('history_uid',
|
||||
existing_type=sa.TEXT(),
|
||||
nullable=True)
|
||||
|
||||
with op.batch_alter_table('message', schema=None) as batch_op:
|
||||
batch_op.alter_column('message',
|
||||
existing_type=sa.TEXT(),
|
||||
nullable=True)
|
||||
batch_op.alter_column('history_uid',
|
||||
existing_type=sa.TEXT(),
|
||||
nullable=True)
|
||||
|
||||
with op.batch_alter_table('history', schema=None) as batch_op:
|
||||
batch_op.drop_constraint('uq_origin_id', type_='unique')
|
||||
batch_op.drop_column('origin_id')
|
||||
|
|
@ -0,0 +1,410 @@
|
|||
"""drop message_types table + fix nullable + rename constraints
|
||||
|
||||
Revision ID: 602caf848068
|
||||
Revises:
|
||||
Create Date: 2021-06-26 12:42:54.148313
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import (
|
||||
Table,
|
||||
Column,
|
||||
MetaData,
|
||||
TEXT,
|
||||
INTEGER,
|
||||
Text,
|
||||
Integer,
|
||||
Float,
|
||||
Enum,
|
||||
ForeignKey,
|
||||
Index,
|
||||
PrimaryKeyConstraint,
|
||||
)
|
||||
from sqlalchemy.sql import table, column
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "602caf848068"
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# we have to recreate former tables for batch_alter_table's reflexion, otherwise the
|
||||
# database will be used, and this will keep unammed UNIQUE constraints in addition
|
||||
# to the named ones that we create
|
||||
metadata = MetaData(
|
||||
naming_convention={
|
||||
"ix": "ix_%(column_0_label)s",
|
||||
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
"pk": "pk_%(table_name)s",
|
||||
},
|
||||
)
|
||||
|
||||
old_profiles_table = Table(
|
||||
"profiles",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
|
||||
Column("name", Text, unique=True),
|
||||
)
|
||||
|
||||
old_components_table = Table(
|
||||
"components",
|
||||
metadata,
|
||||
Column(
|
||||
"profile_id",
|
||||
ForeignKey("profiles.id", ondelete="CASCADE"),
|
||||
nullable=True,
|
||||
primary_key=True,
|
||||
),
|
||||
Column("entry_point", Text, nullable=False),
|
||||
)
|
||||
|
||||
old_message_table = Table(
|
||||
"message",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
|
||||
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
|
||||
Column("message", Text),
|
||||
Column("language", Text),
|
||||
Index("message__history_uid", "history_uid"),
|
||||
)
|
||||
|
||||
old_subject_table = Table(
|
||||
"subject",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
|
||||
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
|
||||
Column("subject", Text),
|
||||
Column("language", Text),
|
||||
Index("subject__history_uid", "history_uid"),
|
||||
)
|
||||
|
||||
old_thread_table = Table(
|
||||
"thread",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
|
||||
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
|
||||
Column("thread_id", Text),
|
||||
Column("parent_id", Text),
|
||||
Index("thread__history_uid", "history_uid"),
|
||||
)
|
||||
|
||||
old_history_table = Table(
|
||||
"history",
|
||||
metadata,
|
||||
Column("uid", Text, primary_key=True, nullable=True),
|
||||
Column("stanza_id", Text),
|
||||
Column("update_uid", Text),
|
||||
Column("profile_id", Integer, ForeignKey("profiles.id", ondelete="CASCADE")),
|
||||
Column("source", Text),
|
||||
Column("dest", Text),
|
||||
Column("source_res", Text),
|
||||
Column("dest_res", Text),
|
||||
Column("timestamp", Float, nullable=False),
|
||||
Column("received_timestamp", Float),
|
||||
Column("type", Text, ForeignKey("message_types.type")),
|
||||
Column("extra", Text),
|
||||
Index("history__profile_id_timestamp", "profile_id", "timestamp"),
|
||||
Index(
|
||||
"history__profile_id_received_timestamp", "profile_id", "received_timestamp"
|
||||
),
|
||||
)
|
||||
|
||||
old_param_gen_table = Table(
|
||||
"param_gen",
|
||||
metadata,
|
||||
Column("category", Text, primary_key=True),
|
||||
Column("name", Text, primary_key=True),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_param_ind_table = Table(
|
||||
"param_ind",
|
||||
metadata,
|
||||
Column("category", Text, primary_key=True),
|
||||
Column("name", Text, primary_key=True),
|
||||
Column(
|
||||
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
|
||||
),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_private_gen_table = Table(
|
||||
"private_gen",
|
||||
metadata,
|
||||
Column("namespace", Text, primary_key=True),
|
||||
Column("key", Text, primary_key=True),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_private_ind_table = Table(
|
||||
"private_ind",
|
||||
metadata,
|
||||
Column("namespace", Text, primary_key=True),
|
||||
Column("key", Text, primary_key=True),
|
||||
Column(
|
||||
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
|
||||
),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_private_gen_bin_table = Table(
|
||||
"private_gen_bin",
|
||||
metadata,
|
||||
Column("namespace", Text, primary_key=True),
|
||||
Column("key", Text, primary_key=True),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_private_ind_bin_table = Table(
|
||||
"private_ind_bin",
|
||||
metadata,
|
||||
Column("namespace", Text, primary_key=True),
|
||||
Column("key", Text, primary_key=True),
|
||||
Column(
|
||||
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
|
||||
),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_files_table = Table(
|
||||
"files",
|
||||
metadata,
|
||||
Column("id", Text, primary_key=True),
|
||||
Column("public_id", Text, unique=True),
|
||||
Column("version", Text, primary_key=True),
|
||||
Column("parent", Text, nullable=False),
|
||||
Column(
|
||||
"type",
|
||||
Enum("file", "directory", name="file_type", create_constraint=True),
|
||||
nullable=False,
|
||||
server_default="file",
|
||||
),
|
||||
Column("file_hash", Text),
|
||||
Column("hash_algo", Text),
|
||||
Column("name", Text, nullable=False),
|
||||
Column("size", Integer),
|
||||
Column("namespace", Text),
|
||||
Column("media_type", Text),
|
||||
Column("media_subtype", Text),
|
||||
Column("created", Float, nullable=False),
|
||||
Column("modified", Float),
|
||||
Column("owner", Text),
|
||||
Column("access", Text),
|
||||
Column("extra", Text),
|
||||
Column("profile_id", ForeignKey("profiles.id", ondelete="CASCADE")),
|
||||
Index("files__profile_id_owner_parent", "profile_id", "owner", "parent"),
|
||||
Index(
|
||||
"files__profile_id_owner_media_type_media_subtype",
|
||||
"profile_id",
|
||||
"owner",
|
||||
"media_type",
|
||||
"media_subtype",
|
||||
),
|
||||
)
|
||||
|
||||
op.drop_table("message_types")
|
||||
|
||||
with op.batch_alter_table(
|
||||
"profiles", copy_from=old_profiles_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.create_unique_constraint(batch_op.f("uq_profiles_name"), ["name"])
|
||||
|
||||
with op.batch_alter_table(
|
||||
"components",
|
||||
copy_from=old_components_table,
|
||||
naming_convention={
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
},
|
||||
schema=None,
|
||||
) as batch_op:
|
||||
batch_op.create_unique_constraint(batch_op.f("uq_profiles_name"), ["name"])
|
||||
|
||||
with op.batch_alter_table(
|
||||
"history",
|
||||
copy_from=old_history_table,
|
||||
naming_convention={
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
},
|
||||
schema=None,
|
||||
) as batch_op:
|
||||
batch_op.alter_column("uid", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column(
|
||||
"type",
|
||||
type_=Enum(
|
||||
"chat",
|
||||
"error",
|
||||
"groupchat",
|
||||
"headline",
|
||||
"normal",
|
||||
"info",
|
||||
name="message_type",
|
||||
create_constraint=True,
|
||||
),
|
||||
existing_type=TEXT(),
|
||||
nullable=False,
|
||||
)
|
||||
batch_op.create_unique_constraint(
|
||||
batch_op.f("uq_history_profile_id"),
|
||||
["profile_id", "stanza_id", "source", "dest"],
|
||||
)
|
||||
batch_op.drop_constraint("fk_history_type_message_types", type_="foreignkey")
|
||||
|
||||
with op.batch_alter_table(
|
||||
"message", copy_from=old_message_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
|
||||
)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"subject", copy_from=old_subject_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
|
||||
)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"thread", copy_from=old_thread_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
|
||||
)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"param_gen", copy_from=old_param_gen_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("category", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("name", existing_type=TEXT(), nullable=False)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"param_ind", copy_from=old_param_ind_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("category", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("name", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"private_gen", copy_from=old_private_gen_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"private_ind", copy_from=old_private_ind_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"private_gen_bin", copy_from=old_private_gen_bin_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
|
||||
|
||||
# found some invalid rows in local database, maybe old values made during development,
|
||||
# but in doubt we have to delete them
|
||||
op.execute("DELETE FROM private_ind_bin WHERE namespace IS NULL")
|
||||
|
||||
with op.batch_alter_table(
|
||||
"private_ind_bin", copy_from=old_private_ind_bin_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"files", copy_from=old_files_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.create_unique_constraint(batch_op.f("uq_files_public_id"), ["public_id"])
|
||||
batch_op.alter_column(
|
||||
"type",
|
||||
type_=Enum("file", "directory", name="file_type", create_constraint=True),
|
||||
existing_type=Text(),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
# downgrade doesn't restore the exact same state as before upgrade, as it
|
||||
# would be useless and waste of resource to restore broken things such as
|
||||
# anonymous constraints
|
||||
with op.batch_alter_table("thread", schema=None) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
|
||||
)
|
||||
|
||||
with op.batch_alter_table("subject", schema=None) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
|
||||
)
|
||||
|
||||
with op.batch_alter_table("private_ind_bin", schema=None) as batch_op:
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("private_ind", schema=None) as batch_op:
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("private_gen_bin", schema=None) as batch_op:
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("private_gen", schema=None) as batch_op:
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("param_ind", schema=None) as batch_op:
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
|
||||
batch_op.alter_column("name", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("category", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("param_gen", schema=None) as batch_op:
|
||||
batch_op.alter_column("name", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("category", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("message", schema=None) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"message_types",
|
||||
Column("type", TEXT(), nullable=True),
|
||||
PrimaryKeyConstraint("type"),
|
||||
)
|
||||
message_types_table = table("message_types", column("type", TEXT()))
|
||||
op.bulk_insert(
|
||||
message_types_table,
|
||||
[
|
||||
{"type": "chat"},
|
||||
{"type": "error"},
|
||||
{"type": "groupchat"},
|
||||
{"type": "headline"},
|
||||
{"type": "normal"},
|
||||
{"type": "info"},
|
||||
],
|
||||
)
|
||||
|
||||
with op.batch_alter_table("history", schema=None) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"type",
|
||||
type_=TEXT(),
|
||||
existing_type=TEXT(),
|
||||
nullable=True,
|
||||
)
|
||||
batch_op.create_foreign_key(
|
||||
batch_op.f("fk_history_type_message_types"),
|
||||
"message_types",
|
||||
["type"],
|
||||
["type"],
|
||||
)
|
||||
batch_op.alter_column("uid", existing_type=TEXT(), nullable=True)
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
"""create table for pubsub subscriptions
|
||||
|
||||
Revision ID: 79e5f3313fa4
|
||||
Revises: 129ac51807e4
|
||||
Create Date: 2022-03-14 17:15:00.689871
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from libervia.backend.memory.sqla_mapping import JID
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '79e5f3313fa4'
|
||||
down_revision = '129ac51807e4'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table('pubsub_subs',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('node_id', sa.Integer(), nullable=False),
|
||||
sa.Column('subscriber', JID(), nullable=True),
|
||||
sa.Column('state', sa.Enum('SUBSCRIBED', 'PENDING', name='state'), nullable=True),
|
||||
sa.ForeignKeyConstraint(['node_id'], ['pubsub_nodes.id'], name=op.f('fk_pubsub_subs_node_id_pubsub_nodes'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_pubsub_subs')),
|
||||
sa.UniqueConstraint('node_id', 'subscriber', name=op.f('uq_pubsub_subs_node_id'))
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table('pubsub_subs')
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
"""create tables for Pubsub caching
|
||||
|
||||
Revision ID: 8974efc51d22
|
||||
Revises: 602caf848068
|
||||
Create Date: 2021-07-27 16:38:54.658212
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from libervia.backend.memory.sqla_mapping import JID, Xml
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '8974efc51d22'
|
||||
down_revision = '602caf848068'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('pubsub_nodes',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('profile_id', sa.Integer(), nullable=True),
|
||||
sa.Column('service', JID(), nullable=True),
|
||||
sa.Column('name', sa.Text(), nullable=False),
|
||||
sa.Column('subscribed', sa.Boolean(create_constraint=True, name='subscribed_bool'), nullable=False),
|
||||
sa.Column('analyser', sa.Text(), nullable=True),
|
||||
sa.Column('sync_state', sa.Enum('IN_PROGRESS', 'COMPLETED', 'ERROR', 'NO_SYNC', name='sync_state', create_constraint=True), nullable=True),
|
||||
sa.Column('sync_state_updated', sa.Float(), nullable=False),
|
||||
sa.Column('type', sa.Text(), nullable=True),
|
||||
sa.Column('subtype', sa.Text(), nullable=True),
|
||||
sa.Column('extra', sa.JSON(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['profile_id'], ['profiles.id'], name=op.f('fk_pubsub_nodes_profile_id_profiles'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_pubsub_nodes')),
|
||||
sa.UniqueConstraint('profile_id', 'service', 'name', name=op.f('uq_pubsub_nodes_profile_id'))
|
||||
)
|
||||
op.create_table('pubsub_items',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('node_id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.Text(), nullable=False),
|
||||
sa.Column('data', Xml(), nullable=False),
|
||||
sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
|
||||
sa.Column('updated', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
|
||||
sa.Column('parsed', sa.JSON(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['node_id'], ['pubsub_nodes.id'], name=op.f('fk_pubsub_items_node_id_pubsub_nodes'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_pubsub_items')),
|
||||
sa.UniqueConstraint('node_id', 'name', name=op.f('uq_pubsub_items_node_id'))
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('pubsub_items')
|
||||
op.drop_table('pubsub_nodes')
|
||||
# ### end Alembic commands ###
|
||||
0
libervia/backend/memory/migration/versions/__init__.py
Normal file
0
libervia/backend/memory/migration/versions/__init__.py
Normal file
1173
libervia/backend/memory/params.py
Normal file
1173
libervia/backend/memory/params.py
Normal file
File diff suppressed because it is too large
Load diff
317
libervia/backend/memory/persistent.py
Normal file
317
libervia/backend/memory/persistent.py
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
|
||||
# SAT: a jabber client
|
||||
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python import failure
|
||||
from libervia.backend.core.i18n import _
|
||||
from libervia.backend.core.log import getLogger
|
||||
|
||||
|
||||
log = getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryNotInitializedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class PersistentDict:
|
||||
r"""A dictionary which save persistently each value assigned
|
||||
|
||||
/!\ be careful, each assignment means a database write
|
||||
/!\ Memory must be initialised before loading/setting value with instances of this class"""
|
||||
storage = None
|
||||
binary = False
|
||||
|
||||
def __init__(self, namespace, profile=None):
|
||||
"""
|
||||
|
||||
@param namespace: unique namespace for this dictionary
|
||||
@param profile(unicode, None): profile which *MUST* exists, or None for general values
|
||||
"""
|
||||
if not self.storage:
|
||||
log.error(_("PersistentDict can't be used before memory initialisation"))
|
||||
raise MemoryNotInitializedError
|
||||
self._cache = {}
|
||||
self.namespace = namespace
|
||||
self.profile = profile
|
||||
|
||||
def _set_cache(self, data):
|
||||
self._cache = data
|
||||
|
||||
def load(self):
|
||||
"""Load persistent data from storage.
|
||||
|
||||
need to be called before any other operation
|
||||
@return: defers the PersistentDict instance itself
|
||||
"""
|
||||
d = defer.ensureDeferred(self.storage.get_privates(
|
||||
self.namespace, binary=self.binary, profile=self.profile
|
||||
))
|
||||
d.addCallback(self._set_cache)
|
||||
d.addCallback(lambda __: self)
|
||||
return d
|
||||
|
||||
def iteritems(self):
|
||||
return iter(self._cache.items())
|
||||
|
||||
def items(self):
|
||||
return self._cache.items()
|
||||
|
||||
def __repr__(self):
|
||||
return self._cache.__repr__()
|
||||
|
||||
def __str__(self):
|
||||
return self._cache.__str__()
|
||||
|
||||
def __lt__(self, other):
|
||||
return self._cache.__lt__(other)
|
||||
|
||||
def __le__(self, other):
|
||||
return self._cache.__le__(other)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self._cache.__eq__(other)
|
||||
|
||||
def __ne__(self, other):
|
||||
return self._cache.__ne__(other)
|
||||
|
||||
def __gt__(self, other):
|
||||
return self._cache.__gt__(other)
|
||||
|
||||
def __ge__(self, other):
|
||||
return self._cache.__ge__(other)
|
||||
|
||||
def __cmp__(self, other):
|
||||
return self._cache.__cmp__(other)
|
||||
|
||||
def __hash__(self):
|
||||
return self._cache.__hash__()
|
||||
|
||||
def __bool__(self):
|
||||
return self._cache.__len__() != 0
|
||||
|
||||
def __contains__(self, key):
|
||||
return self._cache.__contains__(key)
|
||||
|
||||
def __iter__(self):
|
||||
return self._cache.__iter__()
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self._cache.__getitem__(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
defer.ensureDeferred(
|
||||
self.storage.set_private_value(
|
||||
self.namespace, key, value, self.binary, self.profile
|
||||
)
|
||||
)
|
||||
return self._cache.__setitem__(key, value)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.storage.del_private_value(self.namespace, key, self.binary, self.profile)
|
||||
return self._cache.__delitem__(key)
|
||||
|
||||
def clear(self):
|
||||
"""Delete all values from this namespace"""
|
||||
self._cache.clear()
|
||||
return self.storage.del_private_namespace(self.namespace, self.binary, self.profile)
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self._cache.get(key, default)
|
||||
|
||||
def aset(self, key, value):
|
||||
"""Async set, return a Deferred fired when value is actually stored"""
|
||||
self._cache.__setitem__(key, value)
|
||||
return defer.ensureDeferred(
|
||||
self.storage.set_private_value(
|
||||
self.namespace, key, value, self.binary, self.profile
|
||||
)
|
||||
)
|
||||
|
||||
def adel(self, key):
|
||||
"""Async del, return a Deferred fired when value is actually deleted"""
|
||||
self._cache.__delitem__(key)
|
||||
return self.storage.del_private_value(
|
||||
self.namespace, key, self.binary, self.profile)
|
||||
|
||||
def setdefault(self, key, default):
|
||||
try:
|
||||
return self._cache[key]
|
||||
except:
|
||||
self.__setitem__(key, default)
|
||||
return default
|
||||
|
||||
def force(self, name):
|
||||
"""Force saving of an attribute to storage
|
||||
|
||||
@return: deferred fired when data is actually saved
|
||||
"""
|
||||
return defer.ensureDeferred(
|
||||
self.storage.set_private_value(
|
||||
self.namespace, name, self._cache[name], self.binary, self.profile
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class PersistentBinaryDict(PersistentDict):
|
||||
"""Persistent dict where value can be any python data (instead of string only)"""
|
||||
binary = True
|
||||
|
||||
|
||||
class LazyPersistentBinaryDict(PersistentBinaryDict):
|
||||
r"""PersistentBinaryDict which get key/value when needed
|
||||
|
||||
This Persistent need more database access, it is suitable for largest data,
|
||||
to save memory.
|
||||
/!\ most of methods return a Deferred
|
||||
"""
|
||||
# TODO: missing methods should be implemented using database access
|
||||
# TODO: a cache would be useful (which is deleted after a timeout)
|
||||
|
||||
def load(self):
|
||||
# we show a warning as calling load on LazyPersistentBinaryDict sounds like a code mistake
|
||||
log.warning(_("Calling load on LazyPersistentBinaryDict while it's not needed"))
|
||||
|
||||
def iteritems(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def items(self):
|
||||
d = defer.ensureDeferred(self.storage.get_privates(
|
||||
self.namespace, binary=self.binary, profile=self.profile
|
||||
))
|
||||
d.addCallback(lambda data_dict: data_dict.items())
|
||||
return d
|
||||
|
||||
def all(self):
|
||||
return defer.ensureDeferred(self.storage.get_privates(
|
||||
self.namespace, binary=self.binary, profile=self.profile
|
||||
))
|
||||
|
||||
def __repr__(self):
|
||||
return self.__str__()
|
||||
|
||||
def __str__(self):
|
||||
return "LazyPersistentBinaryDict (namespace: {})".format(self.namespace)
|
||||
|
||||
def __lt__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __le__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __eq__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __ne__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __gt__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __ge__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __cmp__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __hash__(self):
|
||||
return hash(str(self.__class__) + self.namespace + (self.profile or ''))
|
||||
|
||||
def __bool__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def __contains__(self, key):
|
||||
raise NotImplementedError
|
||||
|
||||
def __iter__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def _data2value(self, data, key):
|
||||
try:
|
||||
return data[key]
|
||||
except KeyError as e:
|
||||
# we return a Failure here to avoid the jump
|
||||
# into debugger in debug mode.
|
||||
raise failure.Failure(e)
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""get the value as a Deferred"""
|
||||
d = defer.ensureDeferred(self.storage.get_privates(
|
||||
self.namespace, keys=[key], binary=self.binary, profile=self.profile
|
||||
))
|
||||
d.addCallback(self._data2value, key)
|
||||
return d
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
defer.ensureDeferred(
|
||||
self.storage.set_private_value(
|
||||
self.namespace, key, value, self.binary, self.profile
|
||||
)
|
||||
)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.storage.del_private_value(self.namespace, key, self.binary, self.profile)
|
||||
|
||||
def _default_or_exception(self, failure_, default):
|
||||
failure_.trap(KeyError)
|
||||
return default
|
||||
|
||||
def get(self, key, default=None):
|
||||
d = self.__getitem__(key)
|
||||
d.addErrback(self._default_or_exception, default=default)
|
||||
return d
|
||||
|
||||
def aset(self, key, value):
|
||||
"""Async set, return a Deferred fired when value is actually stored"""
|
||||
# FIXME: redundant with force, force must be removed
|
||||
# XXX: similar as PersistentDict.aset, but doesn't use cache
|
||||
return defer.ensureDeferred(
|
||||
self.storage.set_private_value(
|
||||
self.namespace, key, value, self.binary, self.profile
|
||||
)
|
||||
)
|
||||
|
||||
def adel(self, key):
|
||||
"""Async del, return a Deferred fired when value is actually deleted"""
|
||||
# XXX: similar as PersistentDict.adel, but doesn't use cache
|
||||
return self.storage.del_private_value(
|
||||
self.namespace, key, self.binary, self.profile)
|
||||
|
||||
def setdefault(self, key, default):
|
||||
raise NotImplementedError
|
||||
|
||||
def force(self, name, value):
|
||||
"""Force saving of an attribute to storage
|
||||
|
||||
@param value(object): value is needed for LazyPersistentBinaryDict
|
||||
@return: deferred fired when data is actually saved
|
||||
"""
|
||||
return defer.ensureDeferred(
|
||||
self.storage.set_private_value(
|
||||
self.namespace, name, value, self.binary, self.profile
|
||||
)
|
||||
)
|
||||
|
||||
def remove(self, key):
|
||||
"""Delete a key from sotrage, and return a deferred called when it's done
|
||||
|
||||
@param key(unicode): key to delete
|
||||
@return (D): A deferred fired when delete is done
|
||||
"""
|
||||
return self.storage.del_private_value(self.namespace, key, self.binary, self.profile)
|
||||
1704
libervia/backend/memory/sqla.py
Normal file
1704
libervia/backend/memory/sqla.py
Normal file
File diff suppressed because it is too large
Load diff
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue