merge bookmark 0.9
This commit is contained in:
commit
7b05d97229
15 changed files with 6795 additions and 3196 deletions
|
|
@ -33,7 +33,7 @@ from sat.core.i18n import _
|
|||
from sat.core.log import getLogger
|
||||
from sat.core import exceptions
|
||||
from sat.core.constants import Const as C
|
||||
from sat.memory.sqlite import SqliteStorage
|
||||
from sat.memory.sqla import Storage
|
||||
from sat.memory.persistent import PersistentDict
|
||||
from sat.memory.params import Params
|
||||
from sat.memory.disco import Discovery
|
||||
|
|
@ -223,12 +223,11 @@ class PasswordSessions(ProfileSessions):
|
|||
)
|
||||
|
||||
|
||||
class Memory(object):
|
||||
class Memory:
|
||||
"""This class manage all the persistent information"""
|
||||
|
||||
def __init__(self, host):
|
||||
log.info(_("Memory manager init"))
|
||||
self.initialized = defer.Deferred()
|
||||
self.host = host
|
||||
self._entities_cache = {} # XXX: keep presence/last resource/other data in cache
|
||||
# /!\ an entity is not necessarily in roster
|
||||
|
|
@ -240,19 +239,19 @@ class Memory(object):
|
|||
self.disco = Discovery(host)
|
||||
self.config = tools_config.parseMainConf(log_filenames=True)
|
||||
self._cache_path = Path(self.getConfig("", "local_dir"), C.CACHE_DIR)
|
||||
database_file = os.path.expanduser(
|
||||
os.path.join(self.getConfig("", "local_dir"), C.SAVEFILE_DATABASE)
|
||||
)
|
||||
self.storage = SqliteStorage(database_file, host.version)
|
||||
|
||||
async def initialise(self):
|
||||
self.storage = Storage()
|
||||
await self.storage.initialise()
|
||||
PersistentDict.storage = self.storage
|
||||
self.params = Params(host, self.storage)
|
||||
self.params = Params(self.host, self.storage)
|
||||
log.info(_("Loading default params template"))
|
||||
self.params.load_default_params()
|
||||
d = self.storage.initialized.addCallback(lambda ignore: self.load())
|
||||
await self.load()
|
||||
self.memory_data = PersistentDict("memory")
|
||||
d.addCallback(lambda ignore: self.memory_data.load())
|
||||
d.addCallback(lambda ignore: self.disco.load())
|
||||
d.chainDeferred(self.initialized)
|
||||
await self.memory_data.load()
|
||||
await self.disco.load()
|
||||
|
||||
|
||||
## Configuration ##
|
||||
|
||||
|
|
@ -1129,12 +1128,12 @@ class Memory(object):
|
|||
)
|
||||
|
||||
def asyncGetStringParamA(
|
||||
self, name, category, attr="value", security_limit=C.NO_SECURITY_LIMIT,
|
||||
self, name, category, attribute="value", security_limit=C.NO_SECURITY_LIMIT,
|
||||
profile_key=C.PROF_KEY_NONE):
|
||||
|
||||
profile = self.getProfileName(profile_key)
|
||||
return defer.ensureDeferred(self.params.asyncGetStringParamA(
|
||||
name, category, attr, security_limit, profile
|
||||
name, category, attribute, security_limit, profile
|
||||
))
|
||||
|
||||
def _getParamsUI(self, security_limit, app, extra_s, profile_key):
|
||||
|
|
@ -1168,20 +1167,22 @@ class Memory(object):
|
|||
client = self.host.getClient(profile_key)
|
||||
# we accept any type
|
||||
data = data_format.deserialise(data_s, type_check=None)
|
||||
return self.storage.setPrivateValue(
|
||||
namespace, key, data, binary=True, profile=client.profile)
|
||||
return defer.ensureDeferred(self.storage.setPrivateValue(
|
||||
namespace, key, data, binary=True, profile=client.profile))
|
||||
|
||||
def _privateDataGet(self, namespace, key, profile_key):
|
||||
client = self.host.getClient(profile_key)
|
||||
d = self.storage.getPrivates(
|
||||
namespace, [key], binary=True, profile=client.profile)
|
||||
d = defer.ensureDeferred(
|
||||
self.storage.getPrivates(
|
||||
namespace, [key], binary=True, profile=client.profile)
|
||||
)
|
||||
d.addCallback(lambda data_dict: data_format.serialise(data_dict.get(key)))
|
||||
return d
|
||||
|
||||
def _privateDataDelete(self, namespace, key, profile_key):
|
||||
client = self.host.getClient(profile_key)
|
||||
return self.storage.delPrivateValue(
|
||||
namespace, key, binary=True, profile=client.profile)
|
||||
return defer.ensureDeferred(self.storage.delPrivateValue(
|
||||
namespace, key, binary=True, profile=client.profile))
|
||||
|
||||
## Files ##
|
||||
|
||||
|
|
@ -1249,8 +1250,7 @@ class Memory(object):
|
|||
_("unknown access type: {type}").format(type=perm_type)
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def checkPermissionToRoot(self, client, file_data, peer_jid, perms_to_check):
|
||||
async def checkPermissionToRoot(self, client, file_data, peer_jid, perms_to_check):
|
||||
"""do checkFilePermission on file_data and all its parents until root"""
|
||||
current = file_data
|
||||
while True:
|
||||
|
|
@ -1258,7 +1258,7 @@ class Memory(object):
|
|||
parent = current["parent"]
|
||||
if not parent:
|
||||
break
|
||||
files_data = yield self.getFiles(
|
||||
files_data = await self.getFiles(
|
||||
client, peer_jid=None, file_id=parent, perms_to_check=None
|
||||
)
|
||||
try:
|
||||
|
|
@ -1266,8 +1266,7 @@ class Memory(object):
|
|||
except IndexError:
|
||||
raise exceptions.DataError("Missing parent")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _getParentDir(
|
||||
async def _getParentDir(
|
||||
self, client, path, parent, namespace, owner, peer_jid, perms_to_check
|
||||
):
|
||||
"""Retrieve parent node from a path, or last existing directory
|
||||
|
|
@ -1291,7 +1290,7 @@ class Memory(object):
|
|||
# non existing directories will be created
|
||||
parent = ""
|
||||
for idx, path_elt in enumerate(path_elts):
|
||||
directories = yield self.storage.getFiles(
|
||||
directories = await self.storage.getFiles(
|
||||
client,
|
||||
parent=parent,
|
||||
type_=C.FILE_TYPE_DIRECTORY,
|
||||
|
|
@ -1300,7 +1299,7 @@ class Memory(object):
|
|||
owner=owner,
|
||||
)
|
||||
if not directories:
|
||||
defer.returnValue((parent, path_elts[idx:]))
|
||||
return (parent, path_elts[idx:])
|
||||
# from this point, directories don't exist anymore, we have to create them
|
||||
elif len(directories) > 1:
|
||||
raise exceptions.InternalError(
|
||||
|
|
@ -1310,7 +1309,7 @@ class Memory(object):
|
|||
directory = directories[0]
|
||||
self.checkFilePermission(directory, peer_jid, perms_to_check)
|
||||
parent = directory["id"]
|
||||
defer.returnValue((parent, []))
|
||||
return (parent, [])
|
||||
|
||||
def getFileAffiliations(self, file_data: dict) -> Dict[jid.JID, str]:
|
||||
"""Convert file access to pubsub like affiliations"""
|
||||
|
|
@ -1482,8 +1481,7 @@ class Memory(object):
|
|||
)
|
||||
return peer_jid.userhostJID()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def getFiles(
|
||||
async def getFiles(
|
||||
self, client, peer_jid, file_id=None, version=None, parent=None, path=None,
|
||||
type_=None, file_hash=None, hash_algo=None, name=None, namespace=None,
|
||||
mime_type=None, public_id=None, owner=None, access=None, projection=None,
|
||||
|
|
@ -1534,7 +1532,7 @@ class Memory(object):
|
|||
if path is not None:
|
||||
path = str(path)
|
||||
# permission are checked by _getParentDir
|
||||
parent, remaining_path_elts = yield self._getParentDir(
|
||||
parent, remaining_path_elts = await self._getParentDir(
|
||||
client, path, parent, namespace, owner, peer_jid, perms_to_check
|
||||
)
|
||||
if remaining_path_elts:
|
||||
|
|
@ -1544,16 +1542,16 @@ class Memory(object):
|
|||
if parent and peer_jid:
|
||||
# if parent is given directly and permission check is requested,
|
||||
# we need to check all the parents
|
||||
parent_data = yield self.storage.getFiles(client, file_id=parent)
|
||||
parent_data = await self.storage.getFiles(client, file_id=parent)
|
||||
try:
|
||||
parent_data = parent_data[0]
|
||||
except IndexError:
|
||||
raise exceptions.DataError("mising parent")
|
||||
yield self.checkPermissionToRoot(
|
||||
await self.checkPermissionToRoot(
|
||||
client, parent_data, peer_jid, perms_to_check
|
||||
)
|
||||
|
||||
files = yield self.storage.getFiles(
|
||||
files = await self.storage.getFiles(
|
||||
client,
|
||||
file_id=file_id,
|
||||
version=version,
|
||||
|
|
@ -1576,15 +1574,16 @@ class Memory(object):
|
|||
to_remove = []
|
||||
for file_data in files:
|
||||
try:
|
||||
self.checkFilePermission(file_data, peer_jid, perms_to_check, set_affiliation=True)
|
||||
self.checkFilePermission(
|
||||
file_data, peer_jid, perms_to_check, set_affiliation=True
|
||||
)
|
||||
except exceptions.PermissionError:
|
||||
to_remove.append(file_data)
|
||||
for file_data in to_remove:
|
||||
files.remove(file_data)
|
||||
defer.returnValue(files)
|
||||
return files
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setFile(
|
||||
async def setFile(
|
||||
self, client, name, file_id=None, version="", parent=None, path=None,
|
||||
type_=C.FILE_TYPE_FILE, file_hash=None, hash_algo=None, size=None,
|
||||
namespace=None, mime_type=None, public_id=None, created=None, modified=None,
|
||||
|
|
@ -1666,13 +1665,13 @@ class Memory(object):
|
|||
if path is not None:
|
||||
path = str(path)
|
||||
# _getParentDir will check permissions if peer_jid is set, so we use owner
|
||||
parent, remaining_path_elts = yield self._getParentDir(
|
||||
parent, remaining_path_elts = await self._getParentDir(
|
||||
client, path, parent, namespace, owner, owner, perms_to_check
|
||||
)
|
||||
# if remaining directories don't exist, we have to create them
|
||||
for new_dir in remaining_path_elts:
|
||||
new_dir_id = shortuuid.uuid()
|
||||
yield self.storage.setFile(
|
||||
await self.storage.setFile(
|
||||
client,
|
||||
name=new_dir,
|
||||
file_id=new_dir_id,
|
||||
|
|
@ -1689,7 +1688,7 @@ class Memory(object):
|
|||
elif parent is None:
|
||||
parent = ""
|
||||
|
||||
yield self.storage.setFile(
|
||||
await self.storage.setFile(
|
||||
client,
|
||||
file_id=file_id,
|
||||
version=version,
|
||||
|
|
|
|||
3
sat/memory/migration/README
Normal file
3
sat/memory/migration/README
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
This directory and subdirectories contains Alembic migration scripts.
|
||||
|
||||
Please check Libervia documentation for details.
|
||||
0
sat/memory/migration/__init__.py
Normal file
0
sat/memory/migration/__init__.py
Normal file
89
sat/memory/migration/alembic.ini
Normal file
89
sat/memory/migration/alembic.ini
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts
|
||||
script_location = %(here)s
|
||||
|
||||
# template used to generate migration files
|
||||
# file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory.
|
||||
# prepend_sys_path = .
|
||||
|
||||
# timezone to use when rendering the date
|
||||
# within the migration file as well as the filename.
|
||||
# string value is passed to dateutil.tz.gettz()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the
|
||||
# "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; this defaults
|
||||
# to migration/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path
|
||||
# version_locations = %(here)s/bar %(here)s/bat migration/versions
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
# sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
85
sat/memory/migration/env.py
Normal file
85
sat/memory/migration/env.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import asyncio
|
||||
from logging.config import fileConfig
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from alembic import context
|
||||
from sat.memory import sqla_config
|
||||
from sat.memory.sqla_mapping import Base
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
# from myapp import mymodel
|
||||
# target_metadata = mymodel.Base.metadata
|
||||
target_metadata = Base.metadata
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline():
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
db_config = sqla_config.getDbConfig()
|
||||
context.configure(
|
||||
url=db_config["url"],
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection):
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=True
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def run_migrations_online():
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
db_config = sqla_config.getDbConfig()
|
||||
engine = create_async_engine(
|
||||
db_config["url"],
|
||||
poolclass=pool.NullPool,
|
||||
future=True,
|
||||
)
|
||||
|
||||
async with engine.connect() as connection:
|
||||
await connection.run_sync(do_run_migrations)
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
asyncio.run(run_migrations_online())
|
||||
24
sat/memory/migration/script.py.mako
Normal file
24
sat/memory/migration/script.py.mako
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
branch_labels = ${repr(branch_labels)}
|
||||
depends_on = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade():
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade():
|
||||
${downgrades if downgrades else "pass"}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
"""create virtual table for Full-Text Search
|
||||
|
||||
Revision ID: 129ac51807e4
|
||||
Revises: 8974efc51d22
|
||||
Create Date: 2021-08-13 19:13:54.112538
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '129ac51807e4'
|
||||
down_revision = '8974efc51d22'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
queries = [
|
||||
"CREATE VIRTUAL TABLE pubsub_items_fts "
|
||||
"USING fts5(data, content=pubsub_items, content_rowid=id)",
|
||||
"CREATE TRIGGER pubsub_items_fts_sync_ins AFTER INSERT ON pubsub_items BEGIN"
|
||||
" INSERT INTO pubsub_items_fts(rowid, data) VALUES (new.id, new.data);"
|
||||
"END",
|
||||
"CREATE TRIGGER pubsub_items_fts_sync_del AFTER DELETE ON pubsub_items BEGIN"
|
||||
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) "
|
||||
"VALUES('delete', old.id, old.data);"
|
||||
"END",
|
||||
"CREATE TRIGGER pubsub_items_fts_sync_upd AFTER UPDATE ON pubsub_items BEGIN"
|
||||
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) VALUES"
|
||||
"('delete', old.id, old.data);"
|
||||
" INSERT INTO pubsub_items_fts(rowid, data) VALUES(new.id, new.data);"
|
||||
"END",
|
||||
"INSERT INTO pubsub_items_fts(rowid, data) SELECT id, data from pubsub_items"
|
||||
]
|
||||
for q in queries:
|
||||
op.execute(sa.DDL(q))
|
||||
|
||||
|
||||
def downgrade():
|
||||
queries = [
|
||||
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_ins",
|
||||
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_del",
|
||||
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_upd",
|
||||
"DROP TABLE IF EXISTS pubsub_items_fts",
|
||||
]
|
||||
for q in queries:
|
||||
op.execute(sa.DDL(q))
|
||||
|
|
@ -0,0 +1,410 @@
|
|||
"""drop message_types table + fix nullable + rename constraints
|
||||
|
||||
Revision ID: 602caf848068
|
||||
Revises:
|
||||
Create Date: 2021-06-26 12:42:54.148313
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import (
|
||||
Table,
|
||||
Column,
|
||||
MetaData,
|
||||
TEXT,
|
||||
INTEGER,
|
||||
Text,
|
||||
Integer,
|
||||
Float,
|
||||
Enum,
|
||||
ForeignKey,
|
||||
Index,
|
||||
PrimaryKeyConstraint,
|
||||
)
|
||||
from sqlalchemy.sql import table, column
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "602caf848068"
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# we have to recreate former tables for batch_alter_table's reflexion, otherwise the
|
||||
# database will be used, and this will keep unammed UNIQUE constraints in addition
|
||||
# to the named ones that we create
|
||||
metadata = MetaData(
|
||||
naming_convention={
|
||||
"ix": "ix_%(column_0_label)s",
|
||||
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
"pk": "pk_%(table_name)s",
|
||||
},
|
||||
)
|
||||
|
||||
old_profiles_table = Table(
|
||||
"profiles",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
|
||||
Column("name", Text, unique=True),
|
||||
)
|
||||
|
||||
old_components_table = Table(
|
||||
"components",
|
||||
metadata,
|
||||
Column(
|
||||
"profile_id",
|
||||
ForeignKey("profiles.id", ondelete="CASCADE"),
|
||||
nullable=True,
|
||||
primary_key=True,
|
||||
),
|
||||
Column("entry_point", Text, nullable=False),
|
||||
)
|
||||
|
||||
old_message_table = Table(
|
||||
"message",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
|
||||
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
|
||||
Column("message", Text),
|
||||
Column("language", Text),
|
||||
Index("message__history_uid", "history_uid"),
|
||||
)
|
||||
|
||||
old_subject_table = Table(
|
||||
"subject",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
|
||||
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
|
||||
Column("subject", Text),
|
||||
Column("language", Text),
|
||||
Index("subject__history_uid", "history_uid"),
|
||||
)
|
||||
|
||||
old_thread_table = Table(
|
||||
"thread",
|
||||
metadata,
|
||||
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
|
||||
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
|
||||
Column("thread_id", Text),
|
||||
Column("parent_id", Text),
|
||||
Index("thread__history_uid", "history_uid"),
|
||||
)
|
||||
|
||||
old_history_table = Table(
|
||||
"history",
|
||||
metadata,
|
||||
Column("uid", Text, primary_key=True, nullable=True),
|
||||
Column("stanza_id", Text),
|
||||
Column("update_uid", Text),
|
||||
Column("profile_id", Integer, ForeignKey("profiles.id", ondelete="CASCADE")),
|
||||
Column("source", Text),
|
||||
Column("dest", Text),
|
||||
Column("source_res", Text),
|
||||
Column("dest_res", Text),
|
||||
Column("timestamp", Float, nullable=False),
|
||||
Column("received_timestamp", Float),
|
||||
Column("type", Text, ForeignKey("message_types.type")),
|
||||
Column("extra", Text),
|
||||
Index("history__profile_id_timestamp", "profile_id", "timestamp"),
|
||||
Index(
|
||||
"history__profile_id_received_timestamp", "profile_id", "received_timestamp"
|
||||
),
|
||||
)
|
||||
|
||||
old_param_gen_table = Table(
|
||||
"param_gen",
|
||||
metadata,
|
||||
Column("category", Text, primary_key=True),
|
||||
Column("name", Text, primary_key=True),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_param_ind_table = Table(
|
||||
"param_ind",
|
||||
metadata,
|
||||
Column("category", Text, primary_key=True),
|
||||
Column("name", Text, primary_key=True),
|
||||
Column(
|
||||
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
|
||||
),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_private_gen_table = Table(
|
||||
"private_gen",
|
||||
metadata,
|
||||
Column("namespace", Text, primary_key=True),
|
||||
Column("key", Text, primary_key=True),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_private_ind_table = Table(
|
||||
"private_ind",
|
||||
metadata,
|
||||
Column("namespace", Text, primary_key=True),
|
||||
Column("key", Text, primary_key=True),
|
||||
Column(
|
||||
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
|
||||
),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_private_gen_bin_table = Table(
|
||||
"private_gen_bin",
|
||||
metadata,
|
||||
Column("namespace", Text, primary_key=True),
|
||||
Column("key", Text, primary_key=True),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_private_ind_bin_table = Table(
|
||||
"private_ind_bin",
|
||||
metadata,
|
||||
Column("namespace", Text, primary_key=True),
|
||||
Column("key", Text, primary_key=True),
|
||||
Column(
|
||||
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
|
||||
),
|
||||
Column("value", Text),
|
||||
)
|
||||
|
||||
old_files_table = Table(
|
||||
"files",
|
||||
metadata,
|
||||
Column("id", Text, primary_key=True),
|
||||
Column("public_id", Text, unique=True),
|
||||
Column("version", Text, primary_key=True),
|
||||
Column("parent", Text, nullable=False),
|
||||
Column(
|
||||
"type",
|
||||
Enum("file", "directory", name="file_type", create_constraint=True),
|
||||
nullable=False,
|
||||
server_default="file",
|
||||
),
|
||||
Column("file_hash", Text),
|
||||
Column("hash_algo", Text),
|
||||
Column("name", Text, nullable=False),
|
||||
Column("size", Integer),
|
||||
Column("namespace", Text),
|
||||
Column("media_type", Text),
|
||||
Column("media_subtype", Text),
|
||||
Column("created", Float, nullable=False),
|
||||
Column("modified", Float),
|
||||
Column("owner", Text),
|
||||
Column("access", Text),
|
||||
Column("extra", Text),
|
||||
Column("profile_id", ForeignKey("profiles.id", ondelete="CASCADE")),
|
||||
Index("files__profile_id_owner_parent", "profile_id", "owner", "parent"),
|
||||
Index(
|
||||
"files__profile_id_owner_media_type_media_subtype",
|
||||
"profile_id",
|
||||
"owner",
|
||||
"media_type",
|
||||
"media_subtype",
|
||||
),
|
||||
)
|
||||
|
||||
op.drop_table("message_types")
|
||||
|
||||
with op.batch_alter_table(
|
||||
"profiles", copy_from=old_profiles_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.create_unique_constraint(batch_op.f("uq_profiles_name"), ["name"])
|
||||
|
||||
with op.batch_alter_table(
|
||||
"components",
|
||||
copy_from=old_components_table,
|
||||
naming_convention={
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
},
|
||||
schema=None,
|
||||
) as batch_op:
|
||||
batch_op.create_unique_constraint(batch_op.f("uq_profiles_name"), ["name"])
|
||||
|
||||
with op.batch_alter_table(
|
||||
"history",
|
||||
copy_from=old_history_table,
|
||||
naming_convention={
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
},
|
||||
schema=None,
|
||||
) as batch_op:
|
||||
batch_op.alter_column("uid", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column(
|
||||
"type",
|
||||
type_=Enum(
|
||||
"chat",
|
||||
"error",
|
||||
"groupchat",
|
||||
"headline",
|
||||
"normal",
|
||||
"info",
|
||||
name="message_type",
|
||||
create_constraint=True,
|
||||
),
|
||||
existing_type=TEXT(),
|
||||
nullable=False,
|
||||
)
|
||||
batch_op.create_unique_constraint(
|
||||
batch_op.f("uq_history_profile_id"),
|
||||
["profile_id", "stanza_id", "source", "dest"],
|
||||
)
|
||||
batch_op.drop_constraint("fk_history_type_message_types", type_="foreignkey")
|
||||
|
||||
with op.batch_alter_table(
|
||||
"message", copy_from=old_message_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
|
||||
)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"subject", copy_from=old_subject_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
|
||||
)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"thread", copy_from=old_thread_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
|
||||
)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"param_gen", copy_from=old_param_gen_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("category", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("name", existing_type=TEXT(), nullable=False)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"param_ind", copy_from=old_param_ind_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("category", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("name", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"private_gen", copy_from=old_private_gen_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"private_ind", copy_from=old_private_ind_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"private_gen_bin", copy_from=old_private_gen_bin_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
|
||||
|
||||
# found some invalid rows in local database, maybe old values made during development,
|
||||
# but in doubt we have to delete them
|
||||
op.execute("DELETE FROM private_ind_bin WHERE namespace IS NULL")
|
||||
|
||||
with op.batch_alter_table(
|
||||
"private_ind_bin", copy_from=old_private_ind_bin_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
|
||||
|
||||
with op.batch_alter_table(
|
||||
"files", copy_from=old_files_table, schema=None
|
||||
) as batch_op:
|
||||
batch_op.create_unique_constraint(batch_op.f("uq_files_public_id"), ["public_id"])
|
||||
batch_op.alter_column(
|
||||
"type",
|
||||
type_=Enum("file", "directory", name="file_type", create_constraint=True),
|
||||
existing_type=Text(),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
# downgrade doesn't restore the exact same state as before upgrade, as it
|
||||
# would be useless and waste of resource to restore broken things such as
|
||||
# anonymous constraints
|
||||
with op.batch_alter_table("thread", schema=None) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
|
||||
)
|
||||
|
||||
with op.batch_alter_table("subject", schema=None) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
|
||||
)
|
||||
|
||||
with op.batch_alter_table("private_ind_bin", schema=None) as batch_op:
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("private_ind", schema=None) as batch_op:
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("private_gen_bin", schema=None) as batch_op:
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("private_gen", schema=None) as batch_op:
|
||||
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("param_ind", schema=None) as batch_op:
|
||||
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
|
||||
batch_op.alter_column("name", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("category", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("param_gen", schema=None) as batch_op:
|
||||
batch_op.alter_column("name", existing_type=TEXT(), nullable=True)
|
||||
batch_op.alter_column("category", existing_type=TEXT(), nullable=True)
|
||||
|
||||
with op.batch_alter_table("message", schema=None) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"message_types",
|
||||
Column("type", TEXT(), nullable=True),
|
||||
PrimaryKeyConstraint("type"),
|
||||
)
|
||||
message_types_table = table("message_types", column("type", TEXT()))
|
||||
op.bulk_insert(
|
||||
message_types_table,
|
||||
[
|
||||
{"type": "chat"},
|
||||
{"type": "error"},
|
||||
{"type": "groupchat"},
|
||||
{"type": "headline"},
|
||||
{"type": "normal"},
|
||||
{"type": "info"},
|
||||
],
|
||||
)
|
||||
|
||||
with op.batch_alter_table("history", schema=None) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"type",
|
||||
type_=TEXT(),
|
||||
existing_type=TEXT(),
|
||||
nullable=True,
|
||||
)
|
||||
batch_op.create_foreign_key(
|
||||
batch_op.f("fk_history_type_message_types"),
|
||||
"message_types",
|
||||
["type"],
|
||||
["type"],
|
||||
)
|
||||
batch_op.alter_column("uid", existing_type=TEXT(), nullable=True)
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
"""create tables for Pubsub caching
|
||||
|
||||
Revision ID: 8974efc51d22
|
||||
Revises: 602caf848068
|
||||
Create Date: 2021-07-27 16:38:54.658212
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sat.memory.sqla_mapping import JID, Xml
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '8974efc51d22'
|
||||
down_revision = '602caf848068'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('pubsub_nodes',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('profile_id', sa.Integer(), nullable=True),
|
||||
sa.Column('service', JID(), nullable=True),
|
||||
sa.Column('name', sa.Text(), nullable=False),
|
||||
sa.Column('subscribed', sa.Boolean(create_constraint=True, name='subscribed_bool'), nullable=False),
|
||||
sa.Column('analyser', sa.Text(), nullable=True),
|
||||
sa.Column('sync_state', sa.Enum('IN_PROGRESS', 'COMPLETED', 'ERROR', 'NO_SYNC', name='sync_state', create_constraint=True), nullable=True),
|
||||
sa.Column('sync_state_updated', sa.Float(), nullable=False),
|
||||
sa.Column('type', sa.Text(), nullable=True),
|
||||
sa.Column('subtype', sa.Text(), nullable=True),
|
||||
sa.Column('extra', sa.JSON(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['profile_id'], ['profiles.id'], name=op.f('fk_pubsub_nodes_profile_id_profiles'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_pubsub_nodes')),
|
||||
sa.UniqueConstraint('profile_id', 'service', 'name', name=op.f('uq_pubsub_nodes_profile_id'))
|
||||
)
|
||||
op.create_table('pubsub_items',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('node_id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.Text(), nullable=False),
|
||||
sa.Column('data', Xml(), nullable=False),
|
||||
sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
|
||||
sa.Column('updated', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
|
||||
sa.Column('parsed', sa.JSON(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['node_id'], ['pubsub_nodes.id'], name=op.f('fk_pubsub_items_node_id_pubsub_nodes'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_pubsub_items')),
|
||||
sa.UniqueConstraint('node_id', 'name', name=op.f('uq_pubsub_items_node_id'))
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('pubsub_items')
|
||||
op.drop_table('pubsub_nodes')
|
||||
# ### end Alembic commands ###
|
||||
0
sat/memory/migration/versions/__init__.py
Normal file
0
sat/memory/migration/versions/__init__.py
Normal file
|
|
@ -17,10 +17,13 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python import failure
|
||||
from sat.core.i18n import _
|
||||
from sat.core.log import getLogger
|
||||
|
||||
|
||||
log = getLogger(__name__)
|
||||
from twisted.python import failure
|
||||
|
||||
|
||||
class MemoryNotInitializedError(Exception):
|
||||
|
|
@ -57,7 +60,9 @@ class PersistentDict:
|
|||
need to be called before any other operation
|
||||
@return: defers the PersistentDict instance itself
|
||||
"""
|
||||
d = self.storage.getPrivates(self.namespace, binary=self.binary, profile=self.profile)
|
||||
d = defer.ensureDeferred(self.storage.getPrivates(
|
||||
self.namespace, binary=self.binary, profile=self.profile
|
||||
))
|
||||
d.addCallback(self._setCache)
|
||||
d.addCallback(lambda __: self)
|
||||
return d
|
||||
|
|
@ -111,8 +116,11 @@ class PersistentDict:
|
|||
return self._cache.__getitem__(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self.storage.setPrivateValue(self.namespace, key, value, self.binary,
|
||||
self.profile)
|
||||
defer.ensureDeferred(
|
||||
self.storage.setPrivateValue(
|
||||
self.namespace, key, value, self.binary, self.profile
|
||||
)
|
||||
)
|
||||
return self._cache.__setitem__(key, value)
|
||||
|
||||
def __delitem__(self, key):
|
||||
|
|
@ -130,8 +138,11 @@ class PersistentDict:
|
|||
def aset(self, key, value):
|
||||
"""Async set, return a Deferred fired when value is actually stored"""
|
||||
self._cache.__setitem__(key, value)
|
||||
return self.storage.setPrivateValue(self.namespace, key, value,
|
||||
self.binary, self.profile)
|
||||
return defer.ensureDeferred(
|
||||
self.storage.setPrivateValue(
|
||||
self.namespace, key, value, self.binary, self.profile
|
||||
)
|
||||
)
|
||||
|
||||
def adel(self, key):
|
||||
"""Async del, return a Deferred fired when value is actually deleted"""
|
||||
|
|
@ -151,8 +162,11 @@ class PersistentDict:
|
|||
|
||||
@return: deferred fired when data is actually saved
|
||||
"""
|
||||
return self.storage.setPrivateValue(self.namespace, name, self._cache[name],
|
||||
self.binary, self.profile)
|
||||
return defer.ensureDeferred(
|
||||
self.storage.setPrivateValue(
|
||||
self.namespace, name, self._cache[name], self.binary, self.profile
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class PersistentBinaryDict(PersistentDict):
|
||||
|
|
@ -178,12 +192,16 @@ class LazyPersistentBinaryDict(PersistentBinaryDict):
|
|||
raise NotImplementedError
|
||||
|
||||
def items(self):
|
||||
d = self.storage.getPrivates(self.namespace, binary=self.binary, profile=self.profile)
|
||||
d = defer.ensureDeferred(self.storage.getPrivates(
|
||||
self.namespace, binary=self.binary, profile=self.profile
|
||||
))
|
||||
d.addCallback(lambda data_dict: data_dict.items())
|
||||
return d
|
||||
|
||||
def all(self):
|
||||
return self.storage.getPrivates(self.namespace, binary=self.binary, profile=self.profile)
|
||||
return defer.ensureDeferred(self.storage.getPrivates(
|
||||
self.namespace, binary=self.binary, profile=self.profile
|
||||
))
|
||||
|
||||
def __repr__(self):
|
||||
return self.__str__()
|
||||
|
|
@ -234,14 +252,18 @@ class LazyPersistentBinaryDict(PersistentBinaryDict):
|
|||
|
||||
def __getitem__(self, key):
|
||||
"""get the value as a Deferred"""
|
||||
d = self.storage.getPrivates(self.namespace, keys=[key], binary=self.binary,
|
||||
profile=self.profile)
|
||||
d = defer.ensureDeferred(self.storage.getPrivates(
|
||||
self.namespace, keys=[key], binary=self.binary, profile=self.profile
|
||||
))
|
||||
d.addCallback(self._data2value, key)
|
||||
return d
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self.storage.setPrivateValue(self.namespace, key, value, self.binary,
|
||||
self.profile)
|
||||
defer.ensureDeferred(
|
||||
self.storage.setPrivateValue(
|
||||
self.namespace, key, value, self.binary, self.profile
|
||||
)
|
||||
)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.storage.delPrivateValue(self.namespace, key, self.binary, self.profile)
|
||||
|
|
@ -259,8 +281,11 @@ class LazyPersistentBinaryDict(PersistentBinaryDict):
|
|||
"""Async set, return a Deferred fired when value is actually stored"""
|
||||
# FIXME: redundant with force, force must be removed
|
||||
# XXX: similar as PersistentDict.aset, but doesn't use cache
|
||||
return self.storage.setPrivateValue(self.namespace, key, value,
|
||||
self.binary, self.profile)
|
||||
return defer.ensureDeferred(
|
||||
self.storage.setPrivateValue(
|
||||
self.namespace, key, value, self.binary, self.profile
|
||||
)
|
||||
)
|
||||
|
||||
def adel(self, key):
|
||||
"""Async del, return a Deferred fired when value is actually deleted"""
|
||||
|
|
@ -277,7 +302,11 @@ class LazyPersistentBinaryDict(PersistentBinaryDict):
|
|||
@param value(object): value is needed for LazyPersistentBinaryDict
|
||||
@return: deferred fired when data is actually saved
|
||||
"""
|
||||
return self.storage.setPrivateValue(self.namespace, name, value, self.binary, self.profile)
|
||||
return defer.ensureDeferred(
|
||||
self.storage.setPrivateValue(
|
||||
self.namespace, name, value, self.binary, self.profile
|
||||
)
|
||||
)
|
||||
|
||||
def remove(self, key):
|
||||
"""Delete a key from sotrage, and return a deferred called when it's done
|
||||
|
|
|
|||
1594
sat/memory/sqla.py
Normal file
1594
sat/memory/sqla.py
Normal file
File diff suppressed because it is too large
Load diff
40
sat/memory/sqla_config.py
Normal file
40
sat/memory/sqla_config.py
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Libervia: an XMPP client
|
||||
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
from sat.core.constants import Const as C
|
||||
from sat.tools import config
|
||||
|
||||
|
||||
def getDbConfig() -> dict:
|
||||
"""Get configuration for database
|
||||
|
||||
@return: dict with following keys:
|
||||
- type: only "sqlite" for now
|
||||
- path: path to the sqlite DB
|
||||
"""
|
||||
main_conf = config.parseMainConf()
|
||||
local_dir = Path(config.getConfig(main_conf, "", "local_dir"))
|
||||
database_path = (local_dir / C.SAVEFILE_DATABASE).expanduser()
|
||||
url = f"sqlite+aiosqlite:///{quote(str(database_path))}"
|
||||
return {
|
||||
"type": "sqlite",
|
||||
"path": database_path,
|
||||
"url": url,
|
||||
}
|
||||
576
sat/memory/sqla_mapping.py
Normal file
576
sat/memory/sqla_mapping.py
Normal file
|
|
@ -0,0 +1,576 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Libervia: an XMPP client
|
||||
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import pickle
|
||||
import json
|
||||
from datetime import datetime
|
||||
import time
|
||||
import enum
|
||||
from sqlalchemy import (
|
||||
MetaData, Column, Integer, Text, Float, Boolean, DateTime, Enum, JSON, ForeignKey,
|
||||
UniqueConstraint, Index, DDL, event
|
||||
)
|
||||
|
||||
from sqlalchemy.orm import declarative_base, relationship
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
from sqlalchemy.sql.functions import now
|
||||
from twisted.words.protocols.jabber import jid
|
||||
from wokkel import generic
|
||||
|
||||
|
||||
Base = declarative_base(
|
||||
metadata=MetaData(
|
||||
naming_convention={
|
||||
"ix": 'ix_%(column_0_label)s',
|
||||
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
"pk": "pk_%(table_name)s"
|
||||
}
|
||||
)
|
||||
)
|
||||
# keys which are in message data extra but not stored in extra field this is
|
||||
# because those values are stored in separate fields
|
||||
NOT_IN_EXTRA = ('stanza_id', 'received_timestamp', 'update_uid')
|
||||
|
||||
|
||||
class SyncState(enum.Enum):
|
||||
#: synchronisation is currently in progress
|
||||
IN_PROGRESS = 1
|
||||
#: synchronisation is done
|
||||
COMPLETED = 2
|
||||
#: something wrong happened during synchronisation, won't sync
|
||||
ERROR = 3
|
||||
#: synchronisation won't be done even if a syncing analyser matches
|
||||
NO_SYNC = 4
|
||||
|
||||
|
||||
class LegacyPickle(TypeDecorator):
|
||||
"""Handle troubles with data pickled by former version of SàT
|
||||
|
||||
This type is temporary until we do migration to a proper data type
|
||||
"""
|
||||
# Blob is used on SQLite but gives errors when used here, while Text works fine
|
||||
impl = Text
|
||||
cache_ok = True
|
||||
|
||||
def process_bind_param(self, value, dialect):
|
||||
if value is None:
|
||||
return None
|
||||
return pickle.dumps(value, 0)
|
||||
|
||||
def process_result_value(self, value, dialect):
|
||||
if value is None:
|
||||
return None
|
||||
# value types are inconsistent (probably a consequence of Python 2/3 port
|
||||
# and/or SQLite dynamic typing)
|
||||
try:
|
||||
value = value.encode()
|
||||
except AttributeError:
|
||||
pass
|
||||
# "utf-8" encoding is needed to handle Python 2 pickled data
|
||||
return pickle.loads(value, encoding="utf-8")
|
||||
|
||||
|
||||
class Json(TypeDecorator):
|
||||
"""Handle JSON field in DB independant way"""
|
||||
# Blob is used on SQLite but gives errors when used here, while Text works fine
|
||||
impl = Text
|
||||
cache_ok = True
|
||||
|
||||
def process_bind_param(self, value, dialect):
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value)
|
||||
|
||||
def process_result_value(self, value, dialect):
|
||||
if value is None:
|
||||
return None
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
class JsonDefaultDict(Json):
|
||||
"""Json type which convert NULL to empty dict instead of None"""
|
||||
|
||||
def process_result_value(self, value, dialect):
|
||||
if value is None:
|
||||
return {}
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
class Xml(TypeDecorator):
|
||||
impl = Text
|
||||
cache_ok = True
|
||||
|
||||
def process_bind_param(self, value, dialect):
|
||||
if value is None:
|
||||
return None
|
||||
return value.toXml()
|
||||
|
||||
def process_result_value(self, value, dialect):
|
||||
if value is None:
|
||||
return None
|
||||
return generic.parseXml(value.encode())
|
||||
|
||||
|
||||
class JID(TypeDecorator):
|
||||
"""Store twisted JID in text fields"""
|
||||
impl = Text
|
||||
cache_ok = True
|
||||
|
||||
def process_bind_param(self, value, dialect):
|
||||
if value is None:
|
||||
return None
|
||||
return value.full()
|
||||
|
||||
def process_result_value(self, value, dialect):
|
||||
if value is None:
|
||||
return None
|
||||
return jid.JID(value)
|
||||
|
||||
|
||||
class Profile(Base):
|
||||
__tablename__ = "profiles"
|
||||
|
||||
id = Column(
|
||||
Integer,
|
||||
primary_key=True,
|
||||
nullable=True,
|
||||
)
|
||||
name = Column(Text, unique=True)
|
||||
|
||||
params = relationship("ParamInd", back_populates="profile", passive_deletes=True)
|
||||
private_data = relationship(
|
||||
"PrivateInd", back_populates="profile", passive_deletes=True
|
||||
)
|
||||
private_bin_data = relationship(
|
||||
"PrivateIndBin", back_populates="profile", passive_deletes=True
|
||||
)
|
||||
|
||||
|
||||
class Component(Base):
|
||||
__tablename__ = "components"
|
||||
|
||||
profile_id = Column(
|
||||
ForeignKey("profiles.id", ondelete="CASCADE"),
|
||||
nullable=True,
|
||||
primary_key=True
|
||||
)
|
||||
entry_point = Column(Text, nullable=False)
|
||||
profile = relationship("Profile")
|
||||
|
||||
|
||||
class History(Base):
|
||||
__tablename__ = "history"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("profile_id", "stanza_id", "source", "dest"),
|
||||
Index("history__profile_id_timestamp", "profile_id", "timestamp"),
|
||||
Index(
|
||||
"history__profile_id_received_timestamp", "profile_id", "received_timestamp"
|
||||
)
|
||||
)
|
||||
|
||||
uid = Column(Text, primary_key=True)
|
||||
stanza_id = Column(Text)
|
||||
update_uid = Column(Text)
|
||||
profile_id = Column(ForeignKey("profiles.id", ondelete="CASCADE"))
|
||||
source = Column(Text)
|
||||
dest = Column(Text)
|
||||
source_res = Column(Text)
|
||||
dest_res = Column(Text)
|
||||
timestamp = Column(Float, nullable=False)
|
||||
received_timestamp = Column(Float)
|
||||
type = Column(
|
||||
Enum(
|
||||
"chat",
|
||||
"error",
|
||||
"groupchat",
|
||||
"headline",
|
||||
"normal",
|
||||
# info is not XMPP standard, but used to keep track of info like join/leave
|
||||
# in a MUC
|
||||
"info",
|
||||
name="message_type",
|
||||
create_constraint=True,
|
||||
),
|
||||
nullable=False,
|
||||
)
|
||||
extra = Column(LegacyPickle)
|
||||
|
||||
profile = relationship("Profile")
|
||||
messages = relationship("Message", backref="history", passive_deletes=True)
|
||||
subjects = relationship("Subject", backref="history", passive_deletes=True)
|
||||
thread = relationship(
|
||||
"Thread", uselist=False, back_populates="history", passive_deletes=True
|
||||
)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
source_jid = kwargs.pop("source_jid", None)
|
||||
if source_jid is not None:
|
||||
kwargs["source"] = source_jid.userhost()
|
||||
kwargs["source_res"] = source_jid.resource
|
||||
dest_jid = kwargs.pop("dest_jid", None)
|
||||
if dest_jid is not None:
|
||||
kwargs["dest"] = dest_jid.userhost()
|
||||
kwargs["dest_res"] = dest_jid.resource
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def source_jid(self) -> jid.JID:
|
||||
return jid.JID(f"{self.source}/{self.source_res or ''}")
|
||||
|
||||
@source_jid.setter
|
||||
def source_jid(self, source_jid: jid.JID) -> None:
|
||||
self.source = source_jid.userhost
|
||||
self.source_res = source_jid.resource
|
||||
|
||||
@property
|
||||
def dest_jid(self):
|
||||
return jid.JID(f"{self.dest}/{self.dest_res or ''}")
|
||||
|
||||
@dest_jid.setter
|
||||
def dest_jid(self, dest_jid: jid.JID) -> None:
|
||||
self.dest = dest_jid.userhost
|
||||
self.dest_res = dest_jid.resource
|
||||
|
||||
def __repr__(self):
|
||||
dt = datetime.fromtimestamp(self.timestamp)
|
||||
return f"History<{self.source_jid.full()}->{self.dest_jid.full()} [{dt}]>"
|
||||
|
||||
def serialise(self):
|
||||
extra = self.extra
|
||||
if self.stanza_id is not None:
|
||||
extra["stanza_id"] = self.stanza_id
|
||||
if self.update_uid is not None:
|
||||
extra["update_uid"] = self.update_uid
|
||||
if self.received_timestamp is not None:
|
||||
extra["received_timestamp"] = self.received_timestamp
|
||||
if self.thread is not None:
|
||||
extra["thread"] = self.thread.thread_id
|
||||
if self.thread.parent_id is not None:
|
||||
extra["thread_parent"] = self.thread.parent_id
|
||||
|
||||
|
||||
return {
|
||||
"from": f"{self.source}/{self.source_res}" if self.source_res
|
||||
else self.source,
|
||||
"to": f"{self.dest}/{self.dest_res}" if self.dest_res else self.dest,
|
||||
"uid": self.uid,
|
||||
"message": {m.language or '': m.message for m in self.messages},
|
||||
"subject": {m.language or '': m.subject for m in self.subjects},
|
||||
"type": self.type,
|
||||
"extra": extra,
|
||||
"timestamp": self.timestamp,
|
||||
}
|
||||
|
||||
def as_tuple(self):
|
||||
d = self.serialise()
|
||||
return (
|
||||
d['uid'], d['timestamp'], d['from'], d['to'], d['message'], d['subject'],
|
||||
d['type'], d['extra']
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def debug_collection(history_collection):
|
||||
for idx, history in enumerate(history_collection):
|
||||
history.debug_msg(idx)
|
||||
|
||||
def debug_msg(self, idx=None):
|
||||
"""Print messages"""
|
||||
dt = datetime.fromtimestamp(self.timestamp)
|
||||
if idx is not None:
|
||||
dt = f"({idx}) {dt}"
|
||||
parts = []
|
||||
parts.append(f"[{dt}]<{self.source_jid.full()}->{self.dest_jid.full()}> ")
|
||||
for message in self.messages:
|
||||
if message.language:
|
||||
parts.append(f"[{message.language}] ")
|
||||
parts.append(f"{message.message}\n")
|
||||
print("".join(parts))
|
||||
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = "message"
|
||||
__table_args__ = (
|
||||
Index("message__history_uid", "history_uid"),
|
||||
)
|
||||
|
||||
id = Column(
|
||||
Integer,
|
||||
primary_key=True,
|
||||
)
|
||||
history_uid = Column(ForeignKey("history.uid", ondelete="CASCADE"))
|
||||
message = Column(Text)
|
||||
language = Column(Text)
|
||||
|
||||
def __repr__(self):
|
||||
lang_str = f"[{self.language}]" if self.language else ""
|
||||
msg = f"{self.message[:20]}…" if len(self.message)>20 else self.message
|
||||
content = f"{lang_str}{msg}"
|
||||
return f"Message<{content}>"
|
||||
|
||||
|
||||
class Subject(Base):
|
||||
__tablename__ = "subject"
|
||||
__table_args__ = (
|
||||
Index("subject__history_uid", "history_uid"),
|
||||
)
|
||||
|
||||
id = Column(
|
||||
Integer,
|
||||
primary_key=True,
|
||||
)
|
||||
history_uid = Column(ForeignKey("history.uid", ondelete="CASCADE"))
|
||||
subject = Column(Text)
|
||||
language = Column(Text)
|
||||
|
||||
def __repr__(self):
|
||||
lang_str = f"[{self.language}]" if self.language else ""
|
||||
msg = f"{self.subject[:20]}…" if len(self.subject)>20 else self.subject
|
||||
content = f"{lang_str}{msg}"
|
||||
return f"Subject<{content}>"
|
||||
|
||||
|
||||
class Thread(Base):
|
||||
__tablename__ = "thread"
|
||||
__table_args__ = (
|
||||
Index("thread__history_uid", "history_uid"),
|
||||
)
|
||||
|
||||
id = Column(
|
||||
Integer,
|
||||
primary_key=True,
|
||||
)
|
||||
history_uid = Column(ForeignKey("history.uid", ondelete="CASCADE"))
|
||||
thread_id = Column(Text)
|
||||
parent_id = Column(Text)
|
||||
|
||||
history = relationship("History", uselist=False, back_populates="thread")
|
||||
|
||||
def __repr__(self):
|
||||
return f"Thread<{self.thread_id} [parent: {self.parent_id}]>"
|
||||
|
||||
|
||||
class ParamGen(Base):
|
||||
__tablename__ = "param_gen"
|
||||
|
||||
category = Column(Text, primary_key=True)
|
||||
name = Column(Text, primary_key=True)
|
||||
value = Column(Text)
|
||||
|
||||
|
||||
class ParamInd(Base):
|
||||
__tablename__ = "param_ind"
|
||||
|
||||
category = Column(Text, primary_key=True)
|
||||
name = Column(Text, primary_key=True)
|
||||
profile_id = Column(
|
||||
ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
value = Column(Text)
|
||||
|
||||
profile = relationship("Profile", back_populates="params")
|
||||
|
||||
|
||||
class PrivateGen(Base):
|
||||
__tablename__ = "private_gen"
|
||||
|
||||
namespace = Column(Text, primary_key=True)
|
||||
key = Column(Text, primary_key=True)
|
||||
value = Column(Text)
|
||||
|
||||
|
||||
class PrivateInd(Base):
|
||||
__tablename__ = "private_ind"
|
||||
|
||||
namespace = Column(Text, primary_key=True)
|
||||
key = Column(Text, primary_key=True)
|
||||
profile_id = Column(
|
||||
ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
value = Column(Text)
|
||||
|
||||
profile = relationship("Profile", back_populates="private_data")
|
||||
|
||||
|
||||
class PrivateGenBin(Base):
|
||||
__tablename__ = "private_gen_bin"
|
||||
|
||||
namespace = Column(Text, primary_key=True)
|
||||
key = Column(Text, primary_key=True)
|
||||
value = Column(LegacyPickle)
|
||||
|
||||
|
||||
class PrivateIndBin(Base):
|
||||
__tablename__ = "private_ind_bin"
|
||||
|
||||
namespace = Column(Text, primary_key=True)
|
||||
key = Column(Text, primary_key=True)
|
||||
profile_id = Column(
|
||||
ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
value = Column(LegacyPickle)
|
||||
|
||||
profile = relationship("Profile", back_populates="private_bin_data")
|
||||
|
||||
|
||||
class File(Base):
|
||||
__tablename__ = "files"
|
||||
__table_args__ = (
|
||||
Index("files__profile_id_owner_parent", "profile_id", "owner", "parent"),
|
||||
Index(
|
||||
"files__profile_id_owner_media_type_media_subtype",
|
||||
"profile_id",
|
||||
"owner",
|
||||
"media_type",
|
||||
"media_subtype"
|
||||
)
|
||||
)
|
||||
|
||||
id = Column(Text, primary_key=True)
|
||||
public_id = Column(Text, unique=True)
|
||||
version = Column(Text, primary_key=True)
|
||||
parent = Column(Text, nullable=False)
|
||||
type = Column(
|
||||
Enum(
|
||||
"file", "directory",
|
||||
name="file_type",
|
||||
create_constraint=True
|
||||
),
|
||||
nullable=False,
|
||||
server_default="file",
|
||||
)
|
||||
file_hash = Column(Text)
|
||||
hash_algo = Column(Text)
|
||||
name = Column(Text, nullable=False)
|
||||
size = Column(Integer)
|
||||
namespace = Column(Text)
|
||||
media_type = Column(Text)
|
||||
media_subtype = Column(Text)
|
||||
created = Column(Float, nullable=False)
|
||||
modified = Column(Float)
|
||||
owner = Column(JID)
|
||||
access = Column(JsonDefaultDict)
|
||||
extra = Column(JsonDefaultDict)
|
||||
profile_id = Column(ForeignKey("profiles.id", ondelete="CASCADE"))
|
||||
|
||||
profile = relationship("Profile")
|
||||
|
||||
|
||||
class PubsubNode(Base):
|
||||
__tablename__ = "pubsub_nodes"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("profile_id", "service", "name"),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
profile_id = Column(
|
||||
ForeignKey("profiles.id", ondelete="CASCADE")
|
||||
)
|
||||
service = Column(JID)
|
||||
name = Column(Text, nullable=False)
|
||||
subscribed = Column(
|
||||
Boolean(create_constraint=True, name="subscribed_bool"), nullable=False
|
||||
)
|
||||
analyser = Column(Text)
|
||||
sync_state = Column(
|
||||
Enum(
|
||||
SyncState,
|
||||
name="sync_state",
|
||||
create_constraint=True,
|
||||
),
|
||||
nullable=True
|
||||
)
|
||||
sync_state_updated = Column(
|
||||
Float,
|
||||
nullable=False,
|
||||
default=time.time()
|
||||
)
|
||||
type_ = Column(
|
||||
Text, name="type", nullable=True
|
||||
)
|
||||
subtype = Column(
|
||||
Text, nullable=True
|
||||
)
|
||||
extra = Column(JSON)
|
||||
|
||||
items = relationship("PubsubItem", back_populates="node", passive_deletes=True)
|
||||
|
||||
def __str__(self):
|
||||
return f"Pubsub node {self.name!r} at {self.service}"
|
||||
|
||||
|
||||
class PubsubItem(Base):
|
||||
__tablename__ = "pubsub_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("node_id", "name"),
|
||||
)
|
||||
id = Column(Integer, primary_key=True)
|
||||
node_id = Column(ForeignKey("pubsub_nodes.id", ondelete="CASCADE"), nullable=False)
|
||||
name = Column(Text, nullable=False)
|
||||
data = Column(Xml, nullable=False)
|
||||
created = Column(DateTime, nullable=False, server_default=now())
|
||||
updated = Column(DateTime, nullable=False, server_default=now(), onupdate=now())
|
||||
parsed = Column(JSON)
|
||||
|
||||
node = relationship("PubsubNode", back_populates="items")
|
||||
|
||||
|
||||
## Full-Text Search
|
||||
|
||||
# create
|
||||
|
||||
@event.listens_for(PubsubItem.__table__, "after_create")
|
||||
def fts_create(target, connection, **kw):
|
||||
"""Full-Text Search table creation"""
|
||||
if connection.engine.name == "sqlite":
|
||||
# Using SQLite FTS5
|
||||
queries = [
|
||||
"CREATE VIRTUAL TABLE pubsub_items_fts "
|
||||
"USING fts5(data, content=pubsub_items, content_rowid=id)",
|
||||
"CREATE TRIGGER pubsub_items_fts_sync_ins AFTER INSERT ON pubsub_items BEGIN"
|
||||
" INSERT INTO pubsub_items_fts(rowid, data) VALUES (new.id, new.data);"
|
||||
"END",
|
||||
"CREATE TRIGGER pubsub_items_fts_sync_del AFTER DELETE ON pubsub_items BEGIN"
|
||||
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) "
|
||||
"VALUES('delete', old.id, old.data);"
|
||||
"END",
|
||||
"CREATE TRIGGER pubsub_items_fts_sync_upd AFTER UPDATE ON pubsub_items BEGIN"
|
||||
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) VALUES"
|
||||
"('delete', old.id, old.data);"
|
||||
" INSERT INTO pubsub_items_fts(rowid, data) VALUES(new.id, new.data);"
|
||||
"END"
|
||||
]
|
||||
for q in queries:
|
||||
connection.execute(DDL(q))
|
||||
|
||||
# drop
|
||||
|
||||
@event.listens_for(PubsubItem.__table__, "before_drop")
|
||||
def fts_drop(target, connection, **kw):
|
||||
"Full-Text Search table drop" ""
|
||||
if connection.engine.name == "sqlite":
|
||||
# Using SQLite FTS5
|
||||
queries = [
|
||||
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_ins",
|
||||
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_del",
|
||||
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_upd",
|
||||
"DROP TABLE IF EXISTS pubsub_items_fts",
|
||||
]
|
||||
for q in queries:
|
||||
connection.execute(DDL(q))
|
||||
1766
sat/memory/sqlite.py
1766
sat/memory/sqlite.py
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue