merge bookmark 0.9

This commit is contained in:
Goffi 2021-12-01 16:13:31 +01:00
commit 7b05d97229
15 changed files with 6795 additions and 3196 deletions

View file

@ -33,7 +33,7 @@ from sat.core.i18n import _
from sat.core.log import getLogger
from sat.core import exceptions
from sat.core.constants import Const as C
from sat.memory.sqlite import SqliteStorage
from sat.memory.sqla import Storage
from sat.memory.persistent import PersistentDict
from sat.memory.params import Params
from sat.memory.disco import Discovery
@ -223,12 +223,11 @@ class PasswordSessions(ProfileSessions):
)
class Memory(object):
class Memory:
"""This class manage all the persistent information"""
def __init__(self, host):
log.info(_("Memory manager init"))
self.initialized = defer.Deferred()
self.host = host
self._entities_cache = {} # XXX: keep presence/last resource/other data in cache
# /!\ an entity is not necessarily in roster
@ -240,19 +239,19 @@ class Memory(object):
self.disco = Discovery(host)
self.config = tools_config.parseMainConf(log_filenames=True)
self._cache_path = Path(self.getConfig("", "local_dir"), C.CACHE_DIR)
database_file = os.path.expanduser(
os.path.join(self.getConfig("", "local_dir"), C.SAVEFILE_DATABASE)
)
self.storage = SqliteStorage(database_file, host.version)
async def initialise(self):
self.storage = Storage()
await self.storage.initialise()
PersistentDict.storage = self.storage
self.params = Params(host, self.storage)
self.params = Params(self.host, self.storage)
log.info(_("Loading default params template"))
self.params.load_default_params()
d = self.storage.initialized.addCallback(lambda ignore: self.load())
await self.load()
self.memory_data = PersistentDict("memory")
d.addCallback(lambda ignore: self.memory_data.load())
d.addCallback(lambda ignore: self.disco.load())
d.chainDeferred(self.initialized)
await self.memory_data.load()
await self.disco.load()
## Configuration ##
@ -1129,12 +1128,12 @@ class Memory(object):
)
def asyncGetStringParamA(
self, name, category, attr="value", security_limit=C.NO_SECURITY_LIMIT,
self, name, category, attribute="value", security_limit=C.NO_SECURITY_LIMIT,
profile_key=C.PROF_KEY_NONE):
profile = self.getProfileName(profile_key)
return defer.ensureDeferred(self.params.asyncGetStringParamA(
name, category, attr, security_limit, profile
name, category, attribute, security_limit, profile
))
def _getParamsUI(self, security_limit, app, extra_s, profile_key):
@ -1168,20 +1167,22 @@ class Memory(object):
client = self.host.getClient(profile_key)
# we accept any type
data = data_format.deserialise(data_s, type_check=None)
return self.storage.setPrivateValue(
namespace, key, data, binary=True, profile=client.profile)
return defer.ensureDeferred(self.storage.setPrivateValue(
namespace, key, data, binary=True, profile=client.profile))
def _privateDataGet(self, namespace, key, profile_key):
client = self.host.getClient(profile_key)
d = self.storage.getPrivates(
namespace, [key], binary=True, profile=client.profile)
d = defer.ensureDeferred(
self.storage.getPrivates(
namespace, [key], binary=True, profile=client.profile)
)
d.addCallback(lambda data_dict: data_format.serialise(data_dict.get(key)))
return d
def _privateDataDelete(self, namespace, key, profile_key):
client = self.host.getClient(profile_key)
return self.storage.delPrivateValue(
namespace, key, binary=True, profile=client.profile)
return defer.ensureDeferred(self.storage.delPrivateValue(
namespace, key, binary=True, profile=client.profile))
## Files ##
@ -1249,8 +1250,7 @@ class Memory(object):
_("unknown access type: {type}").format(type=perm_type)
)
@defer.inlineCallbacks
def checkPermissionToRoot(self, client, file_data, peer_jid, perms_to_check):
async def checkPermissionToRoot(self, client, file_data, peer_jid, perms_to_check):
"""do checkFilePermission on file_data and all its parents until root"""
current = file_data
while True:
@ -1258,7 +1258,7 @@ class Memory(object):
parent = current["parent"]
if not parent:
break
files_data = yield self.getFiles(
files_data = await self.getFiles(
client, peer_jid=None, file_id=parent, perms_to_check=None
)
try:
@ -1266,8 +1266,7 @@ class Memory(object):
except IndexError:
raise exceptions.DataError("Missing parent")
@defer.inlineCallbacks
def _getParentDir(
async def _getParentDir(
self, client, path, parent, namespace, owner, peer_jid, perms_to_check
):
"""Retrieve parent node from a path, or last existing directory
@ -1291,7 +1290,7 @@ class Memory(object):
# non existing directories will be created
parent = ""
for idx, path_elt in enumerate(path_elts):
directories = yield self.storage.getFiles(
directories = await self.storage.getFiles(
client,
parent=parent,
type_=C.FILE_TYPE_DIRECTORY,
@ -1300,7 +1299,7 @@ class Memory(object):
owner=owner,
)
if not directories:
defer.returnValue((parent, path_elts[idx:]))
return (parent, path_elts[idx:])
# from this point, directories don't exist anymore, we have to create them
elif len(directories) > 1:
raise exceptions.InternalError(
@ -1310,7 +1309,7 @@ class Memory(object):
directory = directories[0]
self.checkFilePermission(directory, peer_jid, perms_to_check)
parent = directory["id"]
defer.returnValue((parent, []))
return (parent, [])
def getFileAffiliations(self, file_data: dict) -> Dict[jid.JID, str]:
"""Convert file access to pubsub like affiliations"""
@ -1482,8 +1481,7 @@ class Memory(object):
)
return peer_jid.userhostJID()
@defer.inlineCallbacks
def getFiles(
async def getFiles(
self, client, peer_jid, file_id=None, version=None, parent=None, path=None,
type_=None, file_hash=None, hash_algo=None, name=None, namespace=None,
mime_type=None, public_id=None, owner=None, access=None, projection=None,
@ -1534,7 +1532,7 @@ class Memory(object):
if path is not None:
path = str(path)
# permission are checked by _getParentDir
parent, remaining_path_elts = yield self._getParentDir(
parent, remaining_path_elts = await self._getParentDir(
client, path, parent, namespace, owner, peer_jid, perms_to_check
)
if remaining_path_elts:
@ -1544,16 +1542,16 @@ class Memory(object):
if parent and peer_jid:
# if parent is given directly and permission check is requested,
# we need to check all the parents
parent_data = yield self.storage.getFiles(client, file_id=parent)
parent_data = await self.storage.getFiles(client, file_id=parent)
try:
parent_data = parent_data[0]
except IndexError:
raise exceptions.DataError("mising parent")
yield self.checkPermissionToRoot(
await self.checkPermissionToRoot(
client, parent_data, peer_jid, perms_to_check
)
files = yield self.storage.getFiles(
files = await self.storage.getFiles(
client,
file_id=file_id,
version=version,
@ -1576,15 +1574,16 @@ class Memory(object):
to_remove = []
for file_data in files:
try:
self.checkFilePermission(file_data, peer_jid, perms_to_check, set_affiliation=True)
self.checkFilePermission(
file_data, peer_jid, perms_to_check, set_affiliation=True
)
except exceptions.PermissionError:
to_remove.append(file_data)
for file_data in to_remove:
files.remove(file_data)
defer.returnValue(files)
return files
@defer.inlineCallbacks
def setFile(
async def setFile(
self, client, name, file_id=None, version="", parent=None, path=None,
type_=C.FILE_TYPE_FILE, file_hash=None, hash_algo=None, size=None,
namespace=None, mime_type=None, public_id=None, created=None, modified=None,
@ -1666,13 +1665,13 @@ class Memory(object):
if path is not None:
path = str(path)
# _getParentDir will check permissions if peer_jid is set, so we use owner
parent, remaining_path_elts = yield self._getParentDir(
parent, remaining_path_elts = await self._getParentDir(
client, path, parent, namespace, owner, owner, perms_to_check
)
# if remaining directories don't exist, we have to create them
for new_dir in remaining_path_elts:
new_dir_id = shortuuid.uuid()
yield self.storage.setFile(
await self.storage.setFile(
client,
name=new_dir,
file_id=new_dir_id,
@ -1689,7 +1688,7 @@ class Memory(object):
elif parent is None:
parent = ""
yield self.storage.setFile(
await self.storage.setFile(
client,
file_id=file_id,
version=version,

View file

@ -0,0 +1,3 @@
This directory and subdirectories contains Alembic migration scripts.
Please check Libervia documentation for details.

View file

View file

@ -0,0 +1,89 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = %(here)s
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory.
# prepend_sys_path = .
# timezone to use when rendering the date
# within the migration file as well as the filename.
# string value is passed to dateutil.tz.gettz()
# leave blank for localtime
# timezone =
# max length of characters to apply to the
# "slug" field
# truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; this defaults
# to migration/versions. When using multiple version
# directories, initial revisions must be specified with --version-path
# version_locations = %(here)s/bar %(here)s/bat migration/versions
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
# sqlalchemy.url = driver://user:pass@localhost/dbname
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks = black
# black.type = console_scripts
# black.entrypoint = black
# black.options = -l 79 REVISION_SCRIPT_FILENAME
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

View file

@ -0,0 +1,85 @@
import asyncio
from logging.config import fileConfig
from sqlalchemy import pool
from sqlalchemy.ext.asyncio import create_async_engine
from alembic import context
from sat.memory import sqla_config
from sat.memory.sqla_mapping import Base
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = Base.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
db_config = sqla_config.getDbConfig()
context.configure(
url=db_config["url"],
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection):
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=True
)
with context.begin_transaction():
context.run_migrations()
async def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
db_config = sqla_config.getDbConfig()
engine = create_async_engine(
db_config["url"],
poolclass=pool.NullPool,
future=True,
)
async with engine.connect() as connection:
await connection.run_sync(do_run_migrations)
if context.is_offline_mode():
run_migrations_offline()
else:
asyncio.run(run_migrations_online())

View file

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View file

@ -0,0 +1,49 @@
"""create virtual table for Full-Text Search
Revision ID: 129ac51807e4
Revises: 8974efc51d22
Create Date: 2021-08-13 19:13:54.112538
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '129ac51807e4'
down_revision = '8974efc51d22'
branch_labels = None
depends_on = None
def upgrade():
queries = [
"CREATE VIRTUAL TABLE pubsub_items_fts "
"USING fts5(data, content=pubsub_items, content_rowid=id)",
"CREATE TRIGGER pubsub_items_fts_sync_ins AFTER INSERT ON pubsub_items BEGIN"
" INSERT INTO pubsub_items_fts(rowid, data) VALUES (new.id, new.data);"
"END",
"CREATE TRIGGER pubsub_items_fts_sync_del AFTER DELETE ON pubsub_items BEGIN"
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) "
"VALUES('delete', old.id, old.data);"
"END",
"CREATE TRIGGER pubsub_items_fts_sync_upd AFTER UPDATE ON pubsub_items BEGIN"
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) VALUES"
"('delete', old.id, old.data);"
" INSERT INTO pubsub_items_fts(rowid, data) VALUES(new.id, new.data);"
"END",
"INSERT INTO pubsub_items_fts(rowid, data) SELECT id, data from pubsub_items"
]
for q in queries:
op.execute(sa.DDL(q))
def downgrade():
queries = [
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_ins",
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_del",
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_upd",
"DROP TABLE IF EXISTS pubsub_items_fts",
]
for q in queries:
op.execute(sa.DDL(q))

View file

@ -0,0 +1,410 @@
"""drop message_types table + fix nullable + rename constraints
Revision ID: 602caf848068
Revises:
Create Date: 2021-06-26 12:42:54.148313
"""
from alembic import op
from sqlalchemy import (
Table,
Column,
MetaData,
TEXT,
INTEGER,
Text,
Integer,
Float,
Enum,
ForeignKey,
Index,
PrimaryKeyConstraint,
)
from sqlalchemy.sql import table, column
# revision identifiers, used by Alembic.
revision = "602caf848068"
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
# we have to recreate former tables for batch_alter_table's reflexion, otherwise the
# database will be used, and this will keep unammed UNIQUE constraints in addition
# to the named ones that we create
metadata = MetaData(
naming_convention={
"ix": "ix_%(column_0_label)s",
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s",
},
)
old_profiles_table = Table(
"profiles",
metadata,
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
Column("name", Text, unique=True),
)
old_components_table = Table(
"components",
metadata,
Column(
"profile_id",
ForeignKey("profiles.id", ondelete="CASCADE"),
nullable=True,
primary_key=True,
),
Column("entry_point", Text, nullable=False),
)
old_message_table = Table(
"message",
metadata,
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
Column("message", Text),
Column("language", Text),
Index("message__history_uid", "history_uid"),
)
old_subject_table = Table(
"subject",
metadata,
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
Column("subject", Text),
Column("language", Text),
Index("subject__history_uid", "history_uid"),
)
old_thread_table = Table(
"thread",
metadata,
Column("id", Integer, primary_key=True, nullable=True, autoincrement=False),
Column("history_uid", ForeignKey("history.uid", ondelete="CASCADE")),
Column("thread_id", Text),
Column("parent_id", Text),
Index("thread__history_uid", "history_uid"),
)
old_history_table = Table(
"history",
metadata,
Column("uid", Text, primary_key=True, nullable=True),
Column("stanza_id", Text),
Column("update_uid", Text),
Column("profile_id", Integer, ForeignKey("profiles.id", ondelete="CASCADE")),
Column("source", Text),
Column("dest", Text),
Column("source_res", Text),
Column("dest_res", Text),
Column("timestamp", Float, nullable=False),
Column("received_timestamp", Float),
Column("type", Text, ForeignKey("message_types.type")),
Column("extra", Text),
Index("history__profile_id_timestamp", "profile_id", "timestamp"),
Index(
"history__profile_id_received_timestamp", "profile_id", "received_timestamp"
),
)
old_param_gen_table = Table(
"param_gen",
metadata,
Column("category", Text, primary_key=True),
Column("name", Text, primary_key=True),
Column("value", Text),
)
old_param_ind_table = Table(
"param_ind",
metadata,
Column("category", Text, primary_key=True),
Column("name", Text, primary_key=True),
Column(
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
),
Column("value", Text),
)
old_private_gen_table = Table(
"private_gen",
metadata,
Column("namespace", Text, primary_key=True),
Column("key", Text, primary_key=True),
Column("value", Text),
)
old_private_ind_table = Table(
"private_ind",
metadata,
Column("namespace", Text, primary_key=True),
Column("key", Text, primary_key=True),
Column(
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
),
Column("value", Text),
)
old_private_gen_bin_table = Table(
"private_gen_bin",
metadata,
Column("namespace", Text, primary_key=True),
Column("key", Text, primary_key=True),
Column("value", Text),
)
old_private_ind_bin_table = Table(
"private_ind_bin",
metadata,
Column("namespace", Text, primary_key=True),
Column("key", Text, primary_key=True),
Column(
"profile_id", ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
),
Column("value", Text),
)
old_files_table = Table(
"files",
metadata,
Column("id", Text, primary_key=True),
Column("public_id", Text, unique=True),
Column("version", Text, primary_key=True),
Column("parent", Text, nullable=False),
Column(
"type",
Enum("file", "directory", name="file_type", create_constraint=True),
nullable=False,
server_default="file",
),
Column("file_hash", Text),
Column("hash_algo", Text),
Column("name", Text, nullable=False),
Column("size", Integer),
Column("namespace", Text),
Column("media_type", Text),
Column("media_subtype", Text),
Column("created", Float, nullable=False),
Column("modified", Float),
Column("owner", Text),
Column("access", Text),
Column("extra", Text),
Column("profile_id", ForeignKey("profiles.id", ondelete="CASCADE")),
Index("files__profile_id_owner_parent", "profile_id", "owner", "parent"),
Index(
"files__profile_id_owner_media_type_media_subtype",
"profile_id",
"owner",
"media_type",
"media_subtype",
),
)
op.drop_table("message_types")
with op.batch_alter_table(
"profiles", copy_from=old_profiles_table, schema=None
) as batch_op:
batch_op.create_unique_constraint(batch_op.f("uq_profiles_name"), ["name"])
with op.batch_alter_table(
"components",
copy_from=old_components_table,
naming_convention={
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
},
schema=None,
) as batch_op:
batch_op.create_unique_constraint(batch_op.f("uq_profiles_name"), ["name"])
with op.batch_alter_table(
"history",
copy_from=old_history_table,
naming_convention={
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
},
schema=None,
) as batch_op:
batch_op.alter_column("uid", existing_type=TEXT(), nullable=False)
batch_op.alter_column(
"type",
type_=Enum(
"chat",
"error",
"groupchat",
"headline",
"normal",
"info",
name="message_type",
create_constraint=True,
),
existing_type=TEXT(),
nullable=False,
)
batch_op.create_unique_constraint(
batch_op.f("uq_history_profile_id"),
["profile_id", "stanza_id", "source", "dest"],
)
batch_op.drop_constraint("fk_history_type_message_types", type_="foreignkey")
with op.batch_alter_table(
"message", copy_from=old_message_table, schema=None
) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
)
with op.batch_alter_table(
"subject", copy_from=old_subject_table, schema=None
) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
)
with op.batch_alter_table(
"thread", copy_from=old_thread_table, schema=None
) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=False, autoincrement=False
)
with op.batch_alter_table(
"param_gen", copy_from=old_param_gen_table, schema=None
) as batch_op:
batch_op.alter_column("category", existing_type=TEXT(), nullable=False)
batch_op.alter_column("name", existing_type=TEXT(), nullable=False)
with op.batch_alter_table(
"param_ind", copy_from=old_param_ind_table, schema=None
) as batch_op:
batch_op.alter_column("category", existing_type=TEXT(), nullable=False)
batch_op.alter_column("name", existing_type=TEXT(), nullable=False)
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
with op.batch_alter_table(
"private_gen", copy_from=old_private_gen_table, schema=None
) as batch_op:
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
with op.batch_alter_table(
"private_ind", copy_from=old_private_ind_table, schema=None
) as batch_op:
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
with op.batch_alter_table(
"private_gen_bin", copy_from=old_private_gen_bin_table, schema=None
) as batch_op:
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
# found some invalid rows in local database, maybe old values made during development,
# but in doubt we have to delete them
op.execute("DELETE FROM private_ind_bin WHERE namespace IS NULL")
with op.batch_alter_table(
"private_ind_bin", copy_from=old_private_ind_bin_table, schema=None
) as batch_op:
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=False)
batch_op.alter_column("key", existing_type=TEXT(), nullable=False)
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=False)
with op.batch_alter_table(
"files", copy_from=old_files_table, schema=None
) as batch_op:
batch_op.create_unique_constraint(batch_op.f("uq_files_public_id"), ["public_id"])
batch_op.alter_column(
"type",
type_=Enum("file", "directory", name="file_type", create_constraint=True),
existing_type=Text(),
nullable=False,
)
def downgrade():
# downgrade doesn't restore the exact same state as before upgrade, as it
# would be useless and waste of resource to restore broken things such as
# anonymous constraints
with op.batch_alter_table("thread", schema=None) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
)
with op.batch_alter_table("subject", schema=None) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
)
with op.batch_alter_table("private_ind_bin", schema=None) as batch_op:
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("private_ind", schema=None) as batch_op:
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("private_gen_bin", schema=None) as batch_op:
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("private_gen", schema=None) as batch_op:
batch_op.alter_column("key", existing_type=TEXT(), nullable=True)
batch_op.alter_column("namespace", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("param_ind", schema=None) as batch_op:
batch_op.alter_column("profile_id", existing_type=INTEGER(), nullable=True)
batch_op.alter_column("name", existing_type=TEXT(), nullable=True)
batch_op.alter_column("category", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("param_gen", schema=None) as batch_op:
batch_op.alter_column("name", existing_type=TEXT(), nullable=True)
batch_op.alter_column("category", existing_type=TEXT(), nullable=True)
with op.batch_alter_table("message", schema=None) as batch_op:
batch_op.alter_column(
"id", existing_type=INTEGER(), nullable=True, autoincrement=False
)
op.create_table(
"message_types",
Column("type", TEXT(), nullable=True),
PrimaryKeyConstraint("type"),
)
message_types_table = table("message_types", column("type", TEXT()))
op.bulk_insert(
message_types_table,
[
{"type": "chat"},
{"type": "error"},
{"type": "groupchat"},
{"type": "headline"},
{"type": "normal"},
{"type": "info"},
],
)
with op.batch_alter_table("history", schema=None) as batch_op:
batch_op.alter_column(
"type",
type_=TEXT(),
existing_type=TEXT(),
nullable=True,
)
batch_op.create_foreign_key(
batch_op.f("fk_history_type_message_types"),
"message_types",
["type"],
["type"],
)
batch_op.alter_column("uid", existing_type=TEXT(), nullable=True)

View file

@ -0,0 +1,57 @@
"""create tables for Pubsub caching
Revision ID: 8974efc51d22
Revises: 602caf848068
Create Date: 2021-07-27 16:38:54.658212
"""
from alembic import op
import sqlalchemy as sa
from sat.memory.sqla_mapping import JID, Xml
# revision identifiers, used by Alembic.
revision = '8974efc51d22'
down_revision = '602caf848068'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('pubsub_nodes',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('profile_id', sa.Integer(), nullable=True),
sa.Column('service', JID(), nullable=True),
sa.Column('name', sa.Text(), nullable=False),
sa.Column('subscribed', sa.Boolean(create_constraint=True, name='subscribed_bool'), nullable=False),
sa.Column('analyser', sa.Text(), nullable=True),
sa.Column('sync_state', sa.Enum('IN_PROGRESS', 'COMPLETED', 'ERROR', 'NO_SYNC', name='sync_state', create_constraint=True), nullable=True),
sa.Column('sync_state_updated', sa.Float(), nullable=False),
sa.Column('type', sa.Text(), nullable=True),
sa.Column('subtype', sa.Text(), nullable=True),
sa.Column('extra', sa.JSON(), nullable=True),
sa.ForeignKeyConstraint(['profile_id'], ['profiles.id'], name=op.f('fk_pubsub_nodes_profile_id_profiles'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id', name=op.f('pk_pubsub_nodes')),
sa.UniqueConstraint('profile_id', 'service', 'name', name=op.f('uq_pubsub_nodes_profile_id'))
)
op.create_table('pubsub_items',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('node_id', sa.Integer(), nullable=False),
sa.Column('name', sa.Text(), nullable=False),
sa.Column('data', Xml(), nullable=False),
sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
sa.Column('updated', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
sa.Column('parsed', sa.JSON(), nullable=True),
sa.ForeignKeyConstraint(['node_id'], ['pubsub_nodes.id'], name=op.f('fk_pubsub_items_node_id_pubsub_nodes'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id', name=op.f('pk_pubsub_items')),
sa.UniqueConstraint('node_id', 'name', name=op.f('uq_pubsub_items_node_id'))
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('pubsub_items')
op.drop_table('pubsub_nodes')
# ### end Alembic commands ###

View file

@ -17,10 +17,13 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from twisted.internet import defer
from twisted.python import failure
from sat.core.i18n import _
from sat.core.log import getLogger
log = getLogger(__name__)
from twisted.python import failure
class MemoryNotInitializedError(Exception):
@ -57,7 +60,9 @@ class PersistentDict:
need to be called before any other operation
@return: defers the PersistentDict instance itself
"""
d = self.storage.getPrivates(self.namespace, binary=self.binary, profile=self.profile)
d = defer.ensureDeferred(self.storage.getPrivates(
self.namespace, binary=self.binary, profile=self.profile
))
d.addCallback(self._setCache)
d.addCallback(lambda __: self)
return d
@ -111,8 +116,11 @@ class PersistentDict:
return self._cache.__getitem__(key)
def __setitem__(self, key, value):
self.storage.setPrivateValue(self.namespace, key, value, self.binary,
self.profile)
defer.ensureDeferred(
self.storage.setPrivateValue(
self.namespace, key, value, self.binary, self.profile
)
)
return self._cache.__setitem__(key, value)
def __delitem__(self, key):
@ -130,8 +138,11 @@ class PersistentDict:
def aset(self, key, value):
"""Async set, return a Deferred fired when value is actually stored"""
self._cache.__setitem__(key, value)
return self.storage.setPrivateValue(self.namespace, key, value,
self.binary, self.profile)
return defer.ensureDeferred(
self.storage.setPrivateValue(
self.namespace, key, value, self.binary, self.profile
)
)
def adel(self, key):
"""Async del, return a Deferred fired when value is actually deleted"""
@ -151,8 +162,11 @@ class PersistentDict:
@return: deferred fired when data is actually saved
"""
return self.storage.setPrivateValue(self.namespace, name, self._cache[name],
self.binary, self.profile)
return defer.ensureDeferred(
self.storage.setPrivateValue(
self.namespace, name, self._cache[name], self.binary, self.profile
)
)
class PersistentBinaryDict(PersistentDict):
@ -178,12 +192,16 @@ class LazyPersistentBinaryDict(PersistentBinaryDict):
raise NotImplementedError
def items(self):
d = self.storage.getPrivates(self.namespace, binary=self.binary, profile=self.profile)
d = defer.ensureDeferred(self.storage.getPrivates(
self.namespace, binary=self.binary, profile=self.profile
))
d.addCallback(lambda data_dict: data_dict.items())
return d
def all(self):
return self.storage.getPrivates(self.namespace, binary=self.binary, profile=self.profile)
return defer.ensureDeferred(self.storage.getPrivates(
self.namespace, binary=self.binary, profile=self.profile
))
def __repr__(self):
return self.__str__()
@ -234,14 +252,18 @@ class LazyPersistentBinaryDict(PersistentBinaryDict):
def __getitem__(self, key):
"""get the value as a Deferred"""
d = self.storage.getPrivates(self.namespace, keys=[key], binary=self.binary,
profile=self.profile)
d = defer.ensureDeferred(self.storage.getPrivates(
self.namespace, keys=[key], binary=self.binary, profile=self.profile
))
d.addCallback(self._data2value, key)
return d
def __setitem__(self, key, value):
self.storage.setPrivateValue(self.namespace, key, value, self.binary,
self.profile)
defer.ensureDeferred(
self.storage.setPrivateValue(
self.namespace, key, value, self.binary, self.profile
)
)
def __delitem__(self, key):
self.storage.delPrivateValue(self.namespace, key, self.binary, self.profile)
@ -259,8 +281,11 @@ class LazyPersistentBinaryDict(PersistentBinaryDict):
"""Async set, return a Deferred fired when value is actually stored"""
# FIXME: redundant with force, force must be removed
# XXX: similar as PersistentDict.aset, but doesn't use cache
return self.storage.setPrivateValue(self.namespace, key, value,
self.binary, self.profile)
return defer.ensureDeferred(
self.storage.setPrivateValue(
self.namespace, key, value, self.binary, self.profile
)
)
def adel(self, key):
"""Async del, return a Deferred fired when value is actually deleted"""
@ -277,7 +302,11 @@ class LazyPersistentBinaryDict(PersistentBinaryDict):
@param value(object): value is needed for LazyPersistentBinaryDict
@return: deferred fired when data is actually saved
"""
return self.storage.setPrivateValue(self.namespace, name, value, self.binary, self.profile)
return defer.ensureDeferred(
self.storage.setPrivateValue(
self.namespace, name, value, self.binary, self.profile
)
)
def remove(self, key):
"""Delete a key from sotrage, and return a deferred called when it's done

1594
sat/memory/sqla.py Normal file

File diff suppressed because it is too large Load diff

40
sat/memory/sqla_config.py Normal file
View file

@ -0,0 +1,40 @@
#!/usr/bin/env python3
# Libervia: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from pathlib import Path
from urllib.parse import quote
from sat.core.constants import Const as C
from sat.tools import config
def getDbConfig() -> dict:
"""Get configuration for database
@return: dict with following keys:
- type: only "sqlite" for now
- path: path to the sqlite DB
"""
main_conf = config.parseMainConf()
local_dir = Path(config.getConfig(main_conf, "", "local_dir"))
database_path = (local_dir / C.SAVEFILE_DATABASE).expanduser()
url = f"sqlite+aiosqlite:///{quote(str(database_path))}"
return {
"type": "sqlite",
"path": database_path,
"url": url,
}

576
sat/memory/sqla_mapping.py Normal file
View file

@ -0,0 +1,576 @@
#!/usr/bin/env python3
# Libervia: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import pickle
import json
from datetime import datetime
import time
import enum
from sqlalchemy import (
MetaData, Column, Integer, Text, Float, Boolean, DateTime, Enum, JSON, ForeignKey,
UniqueConstraint, Index, DDL, event
)
from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy.types import TypeDecorator
from sqlalchemy.sql.functions import now
from twisted.words.protocols.jabber import jid
from wokkel import generic
Base = declarative_base(
metadata=MetaData(
naming_convention={
"ix": 'ix_%(column_0_label)s',
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s"
}
)
)
# keys which are in message data extra but not stored in extra field this is
# because those values are stored in separate fields
NOT_IN_EXTRA = ('stanza_id', 'received_timestamp', 'update_uid')
class SyncState(enum.Enum):
#: synchronisation is currently in progress
IN_PROGRESS = 1
#: synchronisation is done
COMPLETED = 2
#: something wrong happened during synchronisation, won't sync
ERROR = 3
#: synchronisation won't be done even if a syncing analyser matches
NO_SYNC = 4
class LegacyPickle(TypeDecorator):
"""Handle troubles with data pickled by former version of SàT
This type is temporary until we do migration to a proper data type
"""
# Blob is used on SQLite but gives errors when used here, while Text works fine
impl = Text
cache_ok = True
def process_bind_param(self, value, dialect):
if value is None:
return None
return pickle.dumps(value, 0)
def process_result_value(self, value, dialect):
if value is None:
return None
# value types are inconsistent (probably a consequence of Python 2/3 port
# and/or SQLite dynamic typing)
try:
value = value.encode()
except AttributeError:
pass
# "utf-8" encoding is needed to handle Python 2 pickled data
return pickle.loads(value, encoding="utf-8")
class Json(TypeDecorator):
"""Handle JSON field in DB independant way"""
# Blob is used on SQLite but gives errors when used here, while Text works fine
impl = Text
cache_ok = True
def process_bind_param(self, value, dialect):
if value is None:
return None
return json.dumps(value)
def process_result_value(self, value, dialect):
if value is None:
return None
return json.loads(value)
class JsonDefaultDict(Json):
"""Json type which convert NULL to empty dict instead of None"""
def process_result_value(self, value, dialect):
if value is None:
return {}
return json.loads(value)
class Xml(TypeDecorator):
impl = Text
cache_ok = True
def process_bind_param(self, value, dialect):
if value is None:
return None
return value.toXml()
def process_result_value(self, value, dialect):
if value is None:
return None
return generic.parseXml(value.encode())
class JID(TypeDecorator):
"""Store twisted JID in text fields"""
impl = Text
cache_ok = True
def process_bind_param(self, value, dialect):
if value is None:
return None
return value.full()
def process_result_value(self, value, dialect):
if value is None:
return None
return jid.JID(value)
class Profile(Base):
__tablename__ = "profiles"
id = Column(
Integer,
primary_key=True,
nullable=True,
)
name = Column(Text, unique=True)
params = relationship("ParamInd", back_populates="profile", passive_deletes=True)
private_data = relationship(
"PrivateInd", back_populates="profile", passive_deletes=True
)
private_bin_data = relationship(
"PrivateIndBin", back_populates="profile", passive_deletes=True
)
class Component(Base):
__tablename__ = "components"
profile_id = Column(
ForeignKey("profiles.id", ondelete="CASCADE"),
nullable=True,
primary_key=True
)
entry_point = Column(Text, nullable=False)
profile = relationship("Profile")
class History(Base):
__tablename__ = "history"
__table_args__ = (
UniqueConstraint("profile_id", "stanza_id", "source", "dest"),
Index("history__profile_id_timestamp", "profile_id", "timestamp"),
Index(
"history__profile_id_received_timestamp", "profile_id", "received_timestamp"
)
)
uid = Column(Text, primary_key=True)
stanza_id = Column(Text)
update_uid = Column(Text)
profile_id = Column(ForeignKey("profiles.id", ondelete="CASCADE"))
source = Column(Text)
dest = Column(Text)
source_res = Column(Text)
dest_res = Column(Text)
timestamp = Column(Float, nullable=False)
received_timestamp = Column(Float)
type = Column(
Enum(
"chat",
"error",
"groupchat",
"headline",
"normal",
# info is not XMPP standard, but used to keep track of info like join/leave
# in a MUC
"info",
name="message_type",
create_constraint=True,
),
nullable=False,
)
extra = Column(LegacyPickle)
profile = relationship("Profile")
messages = relationship("Message", backref="history", passive_deletes=True)
subjects = relationship("Subject", backref="history", passive_deletes=True)
thread = relationship(
"Thread", uselist=False, back_populates="history", passive_deletes=True
)
def __init__(self, *args, **kwargs):
source_jid = kwargs.pop("source_jid", None)
if source_jid is not None:
kwargs["source"] = source_jid.userhost()
kwargs["source_res"] = source_jid.resource
dest_jid = kwargs.pop("dest_jid", None)
if dest_jid is not None:
kwargs["dest"] = dest_jid.userhost()
kwargs["dest_res"] = dest_jid.resource
super().__init__(*args, **kwargs)
@property
def source_jid(self) -> jid.JID:
return jid.JID(f"{self.source}/{self.source_res or ''}")
@source_jid.setter
def source_jid(self, source_jid: jid.JID) -> None:
self.source = source_jid.userhost
self.source_res = source_jid.resource
@property
def dest_jid(self):
return jid.JID(f"{self.dest}/{self.dest_res or ''}")
@dest_jid.setter
def dest_jid(self, dest_jid: jid.JID) -> None:
self.dest = dest_jid.userhost
self.dest_res = dest_jid.resource
def __repr__(self):
dt = datetime.fromtimestamp(self.timestamp)
return f"History<{self.source_jid.full()}->{self.dest_jid.full()} [{dt}]>"
def serialise(self):
extra = self.extra
if self.stanza_id is not None:
extra["stanza_id"] = self.stanza_id
if self.update_uid is not None:
extra["update_uid"] = self.update_uid
if self.received_timestamp is not None:
extra["received_timestamp"] = self.received_timestamp
if self.thread is not None:
extra["thread"] = self.thread.thread_id
if self.thread.parent_id is not None:
extra["thread_parent"] = self.thread.parent_id
return {
"from": f"{self.source}/{self.source_res}" if self.source_res
else self.source,
"to": f"{self.dest}/{self.dest_res}" if self.dest_res else self.dest,
"uid": self.uid,
"message": {m.language or '': m.message for m in self.messages},
"subject": {m.language or '': m.subject for m in self.subjects},
"type": self.type,
"extra": extra,
"timestamp": self.timestamp,
}
def as_tuple(self):
d = self.serialise()
return (
d['uid'], d['timestamp'], d['from'], d['to'], d['message'], d['subject'],
d['type'], d['extra']
)
@staticmethod
def debug_collection(history_collection):
for idx, history in enumerate(history_collection):
history.debug_msg(idx)
def debug_msg(self, idx=None):
"""Print messages"""
dt = datetime.fromtimestamp(self.timestamp)
if idx is not None:
dt = f"({idx}) {dt}"
parts = []
parts.append(f"[{dt}]<{self.source_jid.full()}->{self.dest_jid.full()}> ")
for message in self.messages:
if message.language:
parts.append(f"[{message.language}] ")
parts.append(f"{message.message}\n")
print("".join(parts))
class Message(Base):
__tablename__ = "message"
__table_args__ = (
Index("message__history_uid", "history_uid"),
)
id = Column(
Integer,
primary_key=True,
)
history_uid = Column(ForeignKey("history.uid", ondelete="CASCADE"))
message = Column(Text)
language = Column(Text)
def __repr__(self):
lang_str = f"[{self.language}]" if self.language else ""
msg = f"{self.message[:20]}…" if len(self.message)>20 else self.message
content = f"{lang_str}{msg}"
return f"Message<{content}>"
class Subject(Base):
__tablename__ = "subject"
__table_args__ = (
Index("subject__history_uid", "history_uid"),
)
id = Column(
Integer,
primary_key=True,
)
history_uid = Column(ForeignKey("history.uid", ondelete="CASCADE"))
subject = Column(Text)
language = Column(Text)
def __repr__(self):
lang_str = f"[{self.language}]" if self.language else ""
msg = f"{self.subject[:20]}…" if len(self.subject)>20 else self.subject
content = f"{lang_str}{msg}"
return f"Subject<{content}>"
class Thread(Base):
__tablename__ = "thread"
__table_args__ = (
Index("thread__history_uid", "history_uid"),
)
id = Column(
Integer,
primary_key=True,
)
history_uid = Column(ForeignKey("history.uid", ondelete="CASCADE"))
thread_id = Column(Text)
parent_id = Column(Text)
history = relationship("History", uselist=False, back_populates="thread")
def __repr__(self):
return f"Thread<{self.thread_id} [parent: {self.parent_id}]>"
class ParamGen(Base):
__tablename__ = "param_gen"
category = Column(Text, primary_key=True)
name = Column(Text, primary_key=True)
value = Column(Text)
class ParamInd(Base):
__tablename__ = "param_ind"
category = Column(Text, primary_key=True)
name = Column(Text, primary_key=True)
profile_id = Column(
ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
)
value = Column(Text)
profile = relationship("Profile", back_populates="params")
class PrivateGen(Base):
__tablename__ = "private_gen"
namespace = Column(Text, primary_key=True)
key = Column(Text, primary_key=True)
value = Column(Text)
class PrivateInd(Base):
__tablename__ = "private_ind"
namespace = Column(Text, primary_key=True)
key = Column(Text, primary_key=True)
profile_id = Column(
ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
)
value = Column(Text)
profile = relationship("Profile", back_populates="private_data")
class PrivateGenBin(Base):
__tablename__ = "private_gen_bin"
namespace = Column(Text, primary_key=True)
key = Column(Text, primary_key=True)
value = Column(LegacyPickle)
class PrivateIndBin(Base):
__tablename__ = "private_ind_bin"
namespace = Column(Text, primary_key=True)
key = Column(Text, primary_key=True)
profile_id = Column(
ForeignKey("profiles.id", ondelete="CASCADE"), primary_key=True
)
value = Column(LegacyPickle)
profile = relationship("Profile", back_populates="private_bin_data")
class File(Base):
__tablename__ = "files"
__table_args__ = (
Index("files__profile_id_owner_parent", "profile_id", "owner", "parent"),
Index(
"files__profile_id_owner_media_type_media_subtype",
"profile_id",
"owner",
"media_type",
"media_subtype"
)
)
id = Column(Text, primary_key=True)
public_id = Column(Text, unique=True)
version = Column(Text, primary_key=True)
parent = Column(Text, nullable=False)
type = Column(
Enum(
"file", "directory",
name="file_type",
create_constraint=True
),
nullable=False,
server_default="file",
)
file_hash = Column(Text)
hash_algo = Column(Text)
name = Column(Text, nullable=False)
size = Column(Integer)
namespace = Column(Text)
media_type = Column(Text)
media_subtype = Column(Text)
created = Column(Float, nullable=False)
modified = Column(Float)
owner = Column(JID)
access = Column(JsonDefaultDict)
extra = Column(JsonDefaultDict)
profile_id = Column(ForeignKey("profiles.id", ondelete="CASCADE"))
profile = relationship("Profile")
class PubsubNode(Base):
__tablename__ = "pubsub_nodes"
__table_args__ = (
UniqueConstraint("profile_id", "service", "name"),
)
id = Column(Integer, primary_key=True)
profile_id = Column(
ForeignKey("profiles.id", ondelete="CASCADE")
)
service = Column(JID)
name = Column(Text, nullable=False)
subscribed = Column(
Boolean(create_constraint=True, name="subscribed_bool"), nullable=False
)
analyser = Column(Text)
sync_state = Column(
Enum(
SyncState,
name="sync_state",
create_constraint=True,
),
nullable=True
)
sync_state_updated = Column(
Float,
nullable=False,
default=time.time()
)
type_ = Column(
Text, name="type", nullable=True
)
subtype = Column(
Text, nullable=True
)
extra = Column(JSON)
items = relationship("PubsubItem", back_populates="node", passive_deletes=True)
def __str__(self):
return f"Pubsub node {self.name!r} at {self.service}"
class PubsubItem(Base):
__tablename__ = "pubsub_items"
__table_args__ = (
UniqueConstraint("node_id", "name"),
)
id = Column(Integer, primary_key=True)
node_id = Column(ForeignKey("pubsub_nodes.id", ondelete="CASCADE"), nullable=False)
name = Column(Text, nullable=False)
data = Column(Xml, nullable=False)
created = Column(DateTime, nullable=False, server_default=now())
updated = Column(DateTime, nullable=False, server_default=now(), onupdate=now())
parsed = Column(JSON)
node = relationship("PubsubNode", back_populates="items")
## Full-Text Search
# create
@event.listens_for(PubsubItem.__table__, "after_create")
def fts_create(target, connection, **kw):
"""Full-Text Search table creation"""
if connection.engine.name == "sqlite":
# Using SQLite FTS5
queries = [
"CREATE VIRTUAL TABLE pubsub_items_fts "
"USING fts5(data, content=pubsub_items, content_rowid=id)",
"CREATE TRIGGER pubsub_items_fts_sync_ins AFTER INSERT ON pubsub_items BEGIN"
" INSERT INTO pubsub_items_fts(rowid, data) VALUES (new.id, new.data);"
"END",
"CREATE TRIGGER pubsub_items_fts_sync_del AFTER DELETE ON pubsub_items BEGIN"
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) "
"VALUES('delete', old.id, old.data);"
"END",
"CREATE TRIGGER pubsub_items_fts_sync_upd AFTER UPDATE ON pubsub_items BEGIN"
" INSERT INTO pubsub_items_fts(pubsub_items_fts, rowid, data) VALUES"
"('delete', old.id, old.data);"
" INSERT INTO pubsub_items_fts(rowid, data) VALUES(new.id, new.data);"
"END"
]
for q in queries:
connection.execute(DDL(q))
# drop
@event.listens_for(PubsubItem.__table__, "before_drop")
def fts_drop(target, connection, **kw):
"Full-Text Search table drop" ""
if connection.engine.name == "sqlite":
# Using SQLite FTS5
queries = [
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_ins",
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_del",
"DROP TRIGGER IF EXISTS pubsub_items_fts_sync_upd",
"DROP TABLE IF EXISTS pubsub_items_fts",
]
for q in queries:
connection.execute(DDL(q))

File diff suppressed because it is too large Load diff