diff --git a/sat/memory/memory.py b/sat/memory/memory.py index b5029c12..afe5ff16 100644 --- a/sat/memory/memory.py +++ b/sat/memory/memory.py @@ -223,7 +223,7 @@ class PasswordSessions(ProfileSessions): ) -class Memory(object): +class Memory: """This class manage all the persistent information""" def __init__(self, host): @@ -241,10 +241,7 @@ class Memory(object): self._cache_path = Path(self.getConfig("", "local_dir"), C.CACHE_DIR) async def initialise(self): - database_file = os.path.expanduser( - os.path.join(self.getConfig("", "local_dir"), C.SAVEFILE_DATABASE) - ) - self.storage = Storage(database_file, self.host.version) + self.storage = Storage() await self.storage.initialise() PersistentDict.storage = self.storage self.params = Params(self.host, self.storage) diff --git a/sat/memory/migration/README b/sat/memory/migration/README new file mode 100644 index 00000000..2b8ec2e4 --- /dev/null +++ b/sat/memory/migration/README @@ -0,0 +1,3 @@ +This directory and subdirectories contains Alembic migration scripts. + +Please check Libervia documentation for details. diff --git a/sat/memory/migration/__init__.py b/sat/memory/migration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sat/memory/migration/alembic.ini b/sat/memory/migration/alembic.ini new file mode 100644 index 00000000..7ddc6b8b --- /dev/null +++ b/sat/memory/migration/alembic.ini @@ -0,0 +1,89 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +# prepend_sys_path = . + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to migration/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat migration/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/sat/memory/migration/env.py b/sat/memory/migration/env.py new file mode 100644 index 00000000..ef5a0942 --- /dev/null +++ b/sat/memory/migration/env.py @@ -0,0 +1,85 @@ +import asyncio +from logging.config import fileConfig +from sqlalchemy import pool +from sqlalchemy.ext.asyncio import create_async_engine +from alembic import context +from sat.memory import sqla_config +from sat.memory.sqla_mapping import Base + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + db_config = sqla_config.getDbConfig() + context.configure( + url=db_config["url"], + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection): + context.configure( + connection=connection, + target_metadata=target_metadata, + render_as_batch=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +async def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + db_config = sqla_config.getDbConfig() + engine = create_async_engine( + db_config["url"], + poolclass=pool.NullPool, + future=True, + ) + + async with engine.connect() as connection: + await connection.run_sync(do_run_migrations) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + asyncio.run(run_migrations_online()) diff --git a/sat/memory/migration/script.py.mako b/sat/memory/migration/script.py.mako new file mode 100644 index 00000000..2c015630 --- /dev/null +++ b/sat/memory/migration/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/sat/memory/migration/versions/__init__.py b/sat/memory/migration/versions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sat/memory/sqla.py b/sat/memory/sqla.py index 47e7717d..8528da49 100644 --- a/sat/memory/sqla.py +++ b/sat/memory/sqla.py @@ -16,18 +16,22 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import sys import time -from typing import Dict, List, Tuple, Iterable, Any, Callable, Optional -from urllib.parse import quote +import asyncio +from asyncio.subprocess import PIPE from pathlib import Path -from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from typing import Dict, List, Tuple, Iterable, Any, Callable, Optional +from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine, create_async_engine from sqlalchemy.exc import IntegrityError, NoResultFound from sqlalchemy.orm import sessionmaker, subqueryload, contains_eager from sqlalchemy.future import select -from sqlalchemy.engine import Engine +from sqlalchemy.engine import Engine, Connection from sqlalchemy import update, delete, and_, or_, event from sqlalchemy.sql.functions import coalesce, sum as sum_ from sqlalchemy.dialects.sqlite import insert +from alembic import script as al_script, config as al_config +from alembic.runtime import migration as al_migration from twisted.internet import defer from twisted.words.protocols.jabber import jid from sat.core.i18n import _ @@ -36,6 +40,8 @@ from sat.core.log import getLogger from sat.core.constants import Const as C from sat.core.core_types import SatXMPPEntity from sat.tools.utils import aio +from sat.memory import migration +from sat.memory import sqla_config from sat.memory.sqla_mapping import ( NOT_IN_EXTRA, Base, @@ -56,6 +62,7 @@ from sat.memory.sqla_mapping import ( log = getLogger(__name__) +migration_path = Path(migration.__file__).parent @event.listens_for(Engine, "connect") @@ -67,32 +74,95 @@ def set_sqlite_pragma(dbapi_connection, connection_record): class Storage: - def __init__(self, db_filename, sat_version): + def __init__(self): self.initialized = defer.Deferred() - self.filename = Path(db_filename) # we keep cache for the profiles (key: profile name, value: profile id) # profile id to name self.profiles: Dict[int, str] = {} # profile id to component entry point self.components: Dict[int, str] = {} + async def migrateApply(self, *args: str, log_output: bool = False) -> None: + """Do a migration command + + Commands are applied by running Alembic in a subprocess. + Arguments are alembic executables commands + + @param log_output: manage stdout and stderr: + - if False, stdout and stderr are buffered, and logged only in case of error + - if True, stdout and stderr will be logged during the command execution + @raise exceptions.DatabaseError: something went wrong while running the + process + """ + stdout, stderr = 2 * (None,) if log_output else 2 * (PIPE,) + proc = await asyncio.create_subprocess_exec( + sys.executable, "-m", "alembic", *args, + stdout=stdout, stderr=stderr, cwd=migration_path + ) + log_out, log_err = await proc.communicate() + if proc.returncode != 0: + msg = _( + "Can't {operation} database (exit code {exit_code})" + ).format( + operation=args[0], + exit_code=proc.returncode + ) + if log_out or log_err: + msg += f":\nstdout: {log_out.decode()}\nstderr: {log_err.decode()}" + log.error(msg) + + raise exceptions.DatabaseError(msg) + + async def createDB(self, engine: AsyncEngine, db_config: dict) -> None: + """Create a new database + + The database is generated from SQLAlchemy model, then stamped by Alembic + """ + # the dir may not exist if it's not the XDG recommended one + db_config["path"].parent.mkdir(0o700, True, True) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + log.debug("stamping the database") + await self.migrateApply("stamp", "head") + log.debug("stamping done") + + def _checkDBIsUpToDate(self, conn: Connection) -> bool: + al_ini_path = migration_path / "alembic.ini" + al_cfg = al_config.Config(al_ini_path) + directory = al_script.ScriptDirectory.from_config(al_cfg) + context = al_migration.MigrationContext.configure(conn) + return set(context.get_current_heads()) == set(directory.get_heads()) + + async def checkAndUpdateDB(self, engine: AsyncEngine, db_config: dict) -> None: + """Check that database is up-to-date, and update if necessary""" + async with engine.connect() as conn: + up_to_date = await conn.run_sync(self._checkDBIsUpToDate) + if up_to_date: + log.debug("Database is up-to-date") + else: + log.info("Database needs to be updated") + log.info("updating…") + await self.migrateApply("upgrade", "head", log_output=True) + log.info("Database is now up-to-date") + @aio - async def initialise(self): + async def initialise(self) -> None: log.info(_("Connecting database")) + db_config = sqla_config.getDbConfig() engine = create_async_engine( - f"sqlite+aiosqlite:///{quote(str(self.filename))}", + db_config["url"], future=True ) self.session = sessionmaker( engine, expire_on_commit=False, class_=AsyncSession ) - new_base = not self.filename.exists() + new_base = not db_config["path"].exists() if new_base: log.info(_("The database is new, creating the tables")) - # the dir may not exist if it's not the XDG recommended one - self.filename.parent.mkdir(0o700, True, True) - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) + await self.createDB(engine, db_config) + else: + await self.checkAndUpdateDB(engine, db_config) async with self.session() as session: result = await session.execute(select(Profile)) diff --git a/sat/memory/sqla_config.py b/sat/memory/sqla_config.py new file mode 100644 index 00000000..c319540b --- /dev/null +++ b/sat/memory/sqla_config.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# Libervia: an XMPP client +# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from pathlib import Path +from urllib.parse import quote +from sat.core.constants import Const as C +from sat.tools import config + + +def getDbConfig() -> dict: + """Get configuration for database + + @return: dict with following keys: + - type: only "sqlite" for now + - path: path to the sqlite DB + """ + main_conf = config.parseMainConf() + local_dir = Path(config.getConfig(main_conf, "", "local_dir")) + database_path = (local_dir / C.SAVEFILE_DATABASE).expanduser() + url = f"sqlite+aiosqlite:///{quote(str(database_path))}" + return { + "type": "sqlite", + "path": database_path, + "url": url, + } diff --git a/setup.py b/setup.py index eca383b4..1dfde162 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,9 @@ install_requires = [ 'omemo-backend-signal < 0.3', 'pyyaml < 5.5.0', 'sqlalchemy >= 1.4', + 'alembic', 'aiosqlite', - 'txdbus' + 'txdbus', ] extras_require = { @@ -140,6 +141,6 @@ setup( use_scm_version=sat_dev_version if is_dev_version else False, install_requires=install_requires, extras_require=extras_require, - package_data={"sat": ["VERSION"]}, + package_data={"sat": ["VERSION", "memory/migration/alembic.ini"]}, python_requires=">=3.7", -) \ No newline at end of file +)