diff --git a/libervia/backend/plugins/plugin_app_manager_docker/__init__.py b/libervia/backend/plugins/plugin_app_manager_docker/__init__.py index 14ee9ee6..088e5d60 100644 --- a/libervia/backend/plugins/plugin_app_manager_docker/__init__.py +++ b/libervia/backend/plugins/plugin_app_manager_docker/__init__.py @@ -22,6 +22,7 @@ from libervia.backend.core.i18n import _ from libervia.backend.core.constants import Const as C from libervia.backend.core import exceptions from libervia.backend.core.log import getLogger +from libervia.backend.plugins.plugin_misc_app_manager.models import AppManagerBackend from libervia.backend.tools.common import async_process log = getLogger(__name__) @@ -40,7 +41,7 @@ PLUGIN_INFO = { } -class AppManagerDocker: +class AppManagerDocker(AppManagerBackend): name = "docker-compose" discover_path = Path(__file__).parent @@ -52,9 +53,7 @@ class AppManagerDocker: raise exceptions.NotFound( '"docker-compose" executable not found, Docker can\'t be used with ' 'application manager') - self.host = host - self._am = host.plugins['APP_MANAGER'] - self._am.register(self) + super().__init__(host) async def start(self, app_data: dict) -> None: await self._am.start_common(app_data) @@ -83,7 +82,7 @@ class AppManagerDocker: path=str(working_dir), ) - async def compute_expose(self, app_data: dict) -> dict: + async def compute_expose(self, app_data: dict) -> None: working_dir = app_data['_instance_dir_path'] expose = app_data['expose'] ports = expose.get('ports', {}) diff --git a/libervia/backend/plugins/plugin_app_manager_docker/libervia_app_weblate.yaml b/libervia/backend/plugins/plugin_app_manager_docker/libervia_app_weblate.yaml index 23170748..39296eef 100644 --- a/libervia/backend/plugins/plugin_app_manager_docker/libervia_app_weblate.yaml +++ b/libervia/backend/plugins/plugin_app_manager_docker/libervia_app_weblate.yaml @@ -20,14 +20,14 @@ override: WEBLATE_SERVER_EMAIL: !libervia_conf ["", "email_from", "weblate@example.com"] WEBLATE_DEFAULT_FROM_EMAIL: !libervia_conf ["", "email_from", "weblate@example.com"] WEBLATE_SITE_DOMAIN: !libervia_conf ["", "public_url"] - WEBLATE_ADMIN_PASSWORD: !libervia_generate_pwd + WEBLATE_ADMIN_PASSWORD: !libervia_generate_pwd {name: admin} WEBLATE_ADMIN_EMAIL: !libervia_conf ["", "email_admins_list", "", "first"] WEBLATE_ENABLE_HTTPS: !libervia_conf ["", "weblate_enable_https", "1"] volumes: - ./settings-override.py:/app/data/settings-override.py:ro expose: url_prefix: [override, services, weblate, environment, WEBLATE_URL_PREFIX] - front_url: !libervia_param [front_url, /translate] + web_url_path: !libervia_param [front_url_path, /translate] web_label: Translate ports: web: diff --git a/libervia/backend/plugins/plugin_misc_app_manager.py b/libervia/backend/plugins/plugin_misc_app_manager/__init__.py similarity index 83% rename from libervia/backend/plugins/plugin_misc_app_manager.py rename to libervia/backend/plugins/plugin_misc_app_manager/__init__.py index 6c6dae0f..54740a44 100644 --- a/libervia/backend/plugins/plugin_misc_app_manager.py +++ b/libervia/backend/plugins/plugin_misc_app_manager/__init__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Libervia plugin to manage external applications -# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) +# Copyright (C) 2009-2024 Jérôme Poisson (goffi@goffi.org) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,19 +16,26 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from pathlib import Path -from typing import Optional, List, Callable from functools import partial, reduce -import tempfile +from pathlib import Path import secrets import string +import tempfile +import time +from typing import Any, Callable, List, Optional +from urllib.parse import urlparse, urlunparse + import shortuuid from twisted.internet import defer from twisted.python.procutils import which -from libervia.backend.core.i18n import _ +from yaml.constructor import ConstructorError + from libervia.backend.core import exceptions from libervia.backend.core.constants import Const as C +from libervia.backend.core.i18n import _ from libervia.backend.core.log import getLogger +from libervia.backend.memory import persistent +from libervia.backend.plugins.plugin_misc_app_manager.models import AppManagerBackend from libervia.backend.tools.common import data_format from libervia.backend.tools.common import async_process @@ -51,7 +58,7 @@ except ImportError: ) from yaml import Loader, Dumper -from yaml.constructor import ConstructorError + PLUGIN_INFO = { @@ -84,6 +91,10 @@ class AppManager: self._started = {} # instance id to app data map self._instances = {} + + + self.persistent_data = persistent.LazyPersistentBinaryDict("app_manager") + host.bridge.add_method( "applications_list", ".plugin", @@ -142,7 +153,7 @@ class AppManager: f"cleaning temporary directory at {data['_instance_dir_path']}") data['_instance_dir_obj'].cleanup() - def _libervia_conf_constr(self, loader, node): + def _libervia_conf_constr(self, loader, node) -> str: """Get a value from Libervia configuration A list is expected with either "name" of a config parameter, a one or more of @@ -153,6 +164,7 @@ class AppManager: - filter filter can be: - "first": get the first item of the value + - "not": get the opposite value (to be used with booleans) """ config_data = loader.construct_sequence(node) if len(config_data) == 1: @@ -165,8 +177,8 @@ class AppManager: section, name, default, filter_ = config_data else: raise ValueError( - f"invalid !libervia_conf value ({config_data!r}), a list of 1 to 4 items is " - "expected" + f"invalid !libervia_conf value ({config_data!r}), a list of 1 to 4 items " + "is expected" ) value = self.host.memory.config_get(section, name, default) @@ -188,16 +200,45 @@ class AppManager: pass elif filter_ == 'first': value = value[0] + elif filter_ == "not": + value = C.bool(value) + value = C.bool_const(not value).lower() else: raise ValueError(f"unmanaged filter: {filter_}") return value - def _libervia_generate_pwd_constr(self, loader, node): - alphabet = string.ascii_letters + string.digits - return ''.join(secrets.choice(alphabet) for i in range(30)) + def _libervia_generate_pwd_constr(self, loader, node) -> str: + """Generate a password and store it in persistent data. - def _libervia_param_constr(self, loader, node): + If the password has already been generated previously, it is reused. + Mapping arguments are: + ``name`` (str) **required** + Name of the password. Will be used to store it. + ``size`` (int) + Size of the password to generate. Default to 30 + """ + try: + kwargs = loader.construct_mapping(node) + name = kwargs["name"] + except (ConstructorError, KeyError): + raise ValueError( + '!libervia_generate_pwd map arguments is missing. At least "name" ' + "must be specified." + ) + pwd_data_key = f"pwd_{name}" + try: + key = self._app_persistent_data[pwd_data_key] + except KeyError: + alphabet = string.ascii_letters + string.digits + key_size = int(kwargs.get("size", 30)) + key = ''.join(secrets.choice(alphabet) for __ in range(key_size)) + self._app_persistent_data[pwd_data_key] = key + else: + log.debug(f"Re-using existing key for {name!r} password.") + return key + + def _libervia_param_constr(self, loader, node) -> str: """Get a parameter specified when starting the application The value can be either the name of the parameter to get, or a list as @@ -207,6 +248,7 @@ class AppManager: name, default = loader.construct_sequence(node) except ConstructorError: name, default = loader.construct_scalar(node), None + assert self._params is not None return self._params.get(name, default) def register(self, manager): @@ -215,10 +257,10 @@ class AppManager: raise exceptions.ConflictError( f"There is already a manager with the name {name}") self._managers[manager.name] = manager - if hasattr(manager, "discover_path"): + if manager.discover_path is not None: self.discover(manager.discover_path, manager) - def get_manager(self, app_data: dict) -> object: + def get_manager(self, app_data: dict) -> AppManagerBackend: """Get manager instance needed for this app @raise exceptions.DataError: something is wrong with the type @@ -279,10 +321,15 @@ class AppManager: raise ValueError(f"invalid id_type: {id_type!r}") def discover( - self, - dir_path: Path, - manager: Optional = None + self, + dir_path: Path, + manager: AppManagerBackend|None = None ) -> None: + """Search for app configuration file. + + App configuration files must start with [APP_FILE_PREFIX] and have a ``.yaml`` + extension. + """ for file_path in dir_path.glob(f"{APP_FILE_PREFIX}*.yaml"): if manager is None: try: @@ -291,10 +338,10 @@ class AppManager: except (exceptions.DataError, exceptions.NotFound) as e: log.warning( f"Can't parse {file_path}, skipping: {e}") + continue app_name = file_path.stem[len(APP_FILE_PREFIX):].strip().lower() if not app_name: - log.warning( - f"invalid app file name at {file_path}") + log.warning(f"invalid app file name at {file_path}") continue app_dict = self._apps.setdefault(app_name, {}) manager_set = app_dict.setdefault(manager, set()) @@ -386,8 +433,11 @@ class AppManager: f"No application found with the name {app_name!r}" ) log.info(f"starting {app_name!r}") + self._app_persistent_data = await self.persistent_data.get(app_name) or {} + self._app_persistent_data["last_started"] = time.time() started_data = self._started.setdefault(app_name, []) app_data = self.parse(app_file_path, extra) + await self.persistent_data.aset(app_name, self._app_persistent_data) app_data["_started"] = False app_data['_file_path'] = app_file_path app_data['_name_canonical'] = app_name @@ -519,6 +569,15 @@ class AppManager: d.addCallback(lambda d: data_format.serialise(d)) return d + def get_app_data_value(self, path: list[str], data: dict[str, Any]) -> Any: + """Return a value from app data from its path. + + @param path: sequence of keys to get to retrieve the value + @return: requested value + @raise KeyError: Can't find the requested value. + """ + return reduce(lambda l, k: l[k], path, data) + async def get_exposed( self, identifier: str, @@ -541,22 +600,38 @@ class AppManager: for name, value in list(passwords.items()): if isinstance(value, list): # if we have a list, is the sequence of keys leading to the value - # to expose. We use "reduce" to retrieve the desired value + # to expose. try: - passwords[name] = reduce(lambda l, k: l[k], value, app_data) - except Exception as e: + passwords[name] = self.get_app_data_value(value, app_data) + except KeyError: log.warning( f"Can't retrieve exposed value for password {name!r}: {e}") del passwords[name] - url_prefix = expose.get("url_prefix") - if isinstance(url_prefix, list): - try: - expose["url_prefix"] = reduce(lambda l, k: l[k], url_prefix, app_data) - except Exception as e: - log.warning( - f"Can't retrieve exposed value for url_prefix: {e}") - del expose["url_prefix"] + for key in ("url_prefix", "front_url"): + value = expose.get(key) + if isinstance(value, list): + try: + expose[key] = self.get_app_data_value(value, app_data) + except KeyError: + log.warning( + f"Can't retrieve exposed value for {key!r} at {value}" + ) + del expose[key] + + front_url = expose.get("front_url") + if front_url: + # we want to be sure that a scheme is defined, defaulting to ``https`` + parsed_url = urlparse(front_url) + if not parsed_url.scheme: + if not parsed_url.netloc: + path_elt = parsed_url.path.split("/", 1) + parsed_url = parsed_url._replace( + netloc=path_elt[0], + path=f"/{path_elt[1]}" if len(path_elt) > 1 else "" + ) + parsed_url = parsed_url._replace(scheme='https') + expose["front_url"] = urlunparse(parsed_url) if extra.get("skip_compute", False): return expose @@ -586,7 +661,7 @@ class AppManager: prepare = {} if not prepare: - log.debug("Nothing to prepare for {name!r}") + log.debug(f"Nothing to prepare for {name!r}") return for action, value in list(prepare.items()): diff --git a/libervia/backend/plugins/plugin_misc_app_manager/models.py b/libervia/backend/plugins/plugin_misc_app_manager/models.py new file mode 100644 index 00000000..69da84d9 --- /dev/null +++ b/libervia/backend/plugins/plugin_misc_app_manager/models.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +# Libervia plugin for Jingle (XEP-0166) +# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import abc +from pathlib import Path + +from libervia.backend.core.i18n import _ + + +class AppManagerBackend(abc.ABC): + """Abstract class for App Manager.""" + + name: str + discover_path: Path|None = None + + def __init__(self, host) -> None: + """Initialize the App Manager. + + @param host: The host object. + """ + self.host = host + self._am = host.plugins["APP_MANAGER"] + self._am.register(self) + + @abc.abstractmethod + async def start(self, app_data: dict) -> None: + """Start the app. + + @param app_data: A dictionary containing app data. + """ + pass + + @abc.abstractmethod + async def stop(self, app_data: dict) -> None: + """Stop the app. + + @param app_data: A dictionary containing app data. + """ + pass + + @abc.abstractmethod + async def compute_expose(self, app_data: dict) -> None: + """Compute exposed data for the app. + + @param app_data: A dictionary containing app data. + """ + pass