refactoring: renamed sat package to libervia.backend

This commit is contained in:
Goffi 2023-06-02 11:49:51 +02:00
commit d2685ba11b
16 changed files with 1293 additions and 1293 deletions

View file

@ -0,0 +1,60 @@
#!/usr/bin/env python3
# Libervia: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
class ANSI(object):
## ANSI escape sequences ##
RESET = "\033[0m"
NORMAL_WEIGHT = "\033[22m"
FG_BLACK, FG_RED, FG_GREEN, FG_YELLOW, FG_BLUE, FG_MAGENTA, FG_CYAN, FG_WHITE = (
"\033[3%dm" % nb for nb in range(8)
)
BOLD = "\033[1m"
BLINK = "\033[5m"
BLINK_OFF = "\033[25m"
@classmethod
def color(cls, *args):
"""output text using ANSI codes
this method simply merge arguments, and add RESET if is not the last arguments
"""
# XXX: we expect to have at least one argument
if args[-1] != cls.RESET:
args = list(args)
args.append(cls.RESET)
return "".join(args)
try:
tty = sys.stdout.isatty()
except (
AttributeError,
TypeError,
): # FIXME: TypeError is here for Pyjamas, need to be removed
tty = False
if not tty:
#  we don't want ANSI escape codes if we are not outputing to a tty!
for attr in dir(ANSI):
if isinstance(getattr(ANSI, attr), str):
setattr(ANSI, attr, "")
del tty

View file

@ -0,0 +1,144 @@
#!/usr/bin/env python3
# SAT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""tools to launch process in a async way (using Twisted)"""
import os.path
from twisted.internet import defer, reactor, protocol
from twisted.python.failure import Failure
from libervia.backend.core.i18n import _
from libervia.backend.core import exceptions
from libervia.backend.core.log import getLogger
log = getLogger(__name__)
class CommandProtocol(protocol.ProcessProtocol):
"""handle an external command"""
# name of the command (unicode)
name = None
# full path to the command (bytes)
command = None
# True to activate logging of command outputs (bool)
log = False
def __init__(self, deferred, stdin=None):
"""
@param deferred(defer.Deferred): will be called when command is completed
@param stdin(str, None): if not None, will be push to standard input
"""
self._stdin = stdin
self._deferred = deferred
self.data = []
self.err_data = []
@property
def command_name(self):
"""returns command name or empty string if it can't be guessed"""
if self.name is not None:
return self.name
elif self.command is not None:
return os.path.splitext(os.path.basename(self.command))[0].decode('utf-8',
'ignore')
else:
return ''
def connectionMade(self):
if self._stdin is not None:
self.transport.write(self._stdin)
self.transport.closeStdin()
def outReceived(self, data):
if self.log:
log.info(data.decode('utf-8', 'replace'))
self.data.append(data)
def errReceived(self, data):
if self.log:
log.warning(data.decode('utf-8', 'replace'))
self.err_data.append(data)
def processEnded(self, reason):
data = b''.join(self.data)
if (reason.value.exitCode == 0):
log.debug(f'{self.command_name!r} command succeed')
# we don't use "replace" on purpose, we want an exception if decoding
# is not working properly
self._deferred.callback(data)
else:
err_data = b''.join(self.err_data)
msg = (_("Can't complete {name} command (error code: {code}):\n"
"stderr:\n{stderr}\n{stdout}\n")
.format(name = self.command_name,
code = reason.value.exitCode,
stderr= err_data.decode(errors='replace'),
stdout = "stdout: " + data.decode(errors='replace')
if data else '',
))
self._deferred.errback(Failure(exceptions.CommandException(
msg, data, err_data)))
@classmethod
def run(cls, *args, **kwargs):
"""Create a new CommandProtocol and execute the given command.
@param *args(unicode): command arguments
if cls.command is specified, it will be the path to the command to execute
otherwise, first argument must be the path
@param **kwargs: can be:
- stdin(unicode, None): data to push to standard input
- verbose(bool): if True stdout and stderr will be logged
other keyword arguments will be used in reactor.spawnProcess
@return ((D)bytes): stdout in case of success
@raise RuntimeError: command returned a non zero status
stdin and stdout will be given as arguments
"""
stdin = kwargs.pop('stdin', None)
if stdin is not None:
stdin = stdin.encode('utf-8')
verbose = kwargs.pop('verbose', False)
args = list(args)
d = defer.Deferred()
prot = cls(d, stdin=stdin)
if verbose:
prot.log = True
if cls.command is None:
if not args:
raise ValueError(
"You must either specify cls.command or use a full path to command "
"to execute as first argument")
command = args.pop(0)
if prot.name is None:
name = os.path.splitext(os.path.basename(command))[0]
prot.name = name
else:
command = cls.command
cmd_args = [command] + args
if "env" not in kwargs:
# we pass parent environment by default
kwargs["env"] = None
reactor.spawnProcess(prot,
command,
cmd_args,
**kwargs)
return d
run = CommandProtocol.run

View file

@ -0,0 +1,53 @@
#!/usr/bin/env python3
# Libervia: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""tools to launch process in a async way (using Twisted)"""
from collections import OrderedDict
from typing import Optional, Callable, Awaitable
from libervia.backend.core.log import getLogger
log = getLogger(__name__)
def async_lru(maxsize: Optional[int] = 50) -> Callable:
"""Decorator to cache async function results using LRU algorithm
@param maxsize: maximum number of items to keep in cache.
None to have no limit
"""
def decorator(func: Callable) -> Callable:
cache = OrderedDict()
async def wrapper(*args) -> Awaitable:
if args in cache:
log.debug(f"using result in cache for {args}")
cache.move_to_end(args)
result = cache[args]
return result
log.debug(f"caching result for {args}")
result = await func(*args)
cache[args] = result
if maxsize is not None and len(cache) > maxsize:
value = cache.popitem(False)
log.debug(f"Removing LRU value: {value}")
return result
return wrapper
return decorator

View file

@ -0,0 +1,153 @@
#!/usr/bin/env python3
# SAT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
""" tools common to backend and frontends """
#  FIXME: json may be more appropriate than manual serialising like done here
from typing import Any
from libervia.backend.core import exceptions
import json
def dict2iter(name, dict_, pop=False):
"""iterate into a list serialised in a dict
name is the name of the key.
Serialisation is done with [name] [name#1] [name#2] and so on
e.g.: if name is 'group', keys are group, group#1, group#2, ...
iteration stop at first missing increment
Empty values are possible
@param name(unicode): name of the key
@param dict_(dict): dictionary with the serialised list
@param pop(bool): if True, remove the value from dict
@return iter: iterate through the deserialised list
"""
if pop:
get = lambda d, k: d.pop(k)
else:
get = lambda d, k: d[k]
try:
yield get(dict_, name)
except KeyError:
return
else:
idx = 1
while True:
try:
yield get(dict_, "{}#{}".format(name, idx))
except KeyError:
return
else:
idx += 1
def dict2iterdict(name, dict_, extra_keys, pop=False):
"""like dict2iter but yield dictionaries
params are like in [dict2iter], extra_keys is used for extra dict keys.
e.g. dict2iterdict(comments, mb_data, ('node', 'service')) will yield dicts like:
{u'comments': u'value1', u'node': u'value2', u'service': u'value3'}
"""
#  FIXME: this format seem overcomplicated, it may be more appropriate to use json here
if pop:
get = lambda d, k: d.pop(k)
else:
get = lambda d, k: d[k]
for idx, main_value in enumerate(dict2iter(name, dict_, pop=pop)):
ret = {name: main_value}
for k in extra_keys:
ret[k] = get(
dict_, "{}{}_{}".format(name, ("#" + str(idx)) if idx else "", k)
)
yield ret
def iter2dict(name, iter_, dict_=None, check_conflict=True):
"""Fill a dict with values from an iterable
name is used to serialise iter_, in the same way as in [dict2iter]
Build from the tags a dict using the microblog data format.
@param name(unicode): key to use for serialisation
e.g. "group" to have keys "group", "group#1", "group#2", ...
@param iter_(iterable): values to store
@param dict_(None, dict): dictionary to fill, or None to create one
@param check_conflict(bool): if True, raise an exception in case of existing key
@return (dict): filled dict, or newly created one
@raise exceptions.ConflictError: a needed key already exists
"""
if dict_ is None:
dict_ = {}
for idx, value in enumerate(iter_):
if idx == 0:
key = name
else:
key = "{}#{}".format(name, idx)
if check_conflict and key in dict_:
raise exceptions.ConflictError
dict_[key] = value
return dict
def get_sub_dict(name, dict_, sep="_"):
"""get a sub dictionary from a serialised dictionary
look for keys starting with name, and create a dict with it
eg.: if "key" is looked for, {'html': 1, 'key_toto': 2, 'key_titi': 3} will return:
{None: 1, toto: 2, titi: 3}
@param name(unicode): name of the key
@param dict_(dict): dictionary with the serialised list
@param sep(unicode): separator used between name and subkey
@return iter: iterate through the deserialised items
"""
for k, v in dict_.items():
if k.startswith(name):
if k == name:
yield None, v
else:
if k[len(name)] != sep:
continue
else:
yield k[len(name) + 1 :], v
def serialise(data):
"""Serialise data so it can be sent to bridge
@return(unicode): serialised data, can be transmitted as string to the bridge
"""
return json.dumps(data, ensure_ascii=False, default=str)
def deserialise(serialised_data: str, default: Any = None, type_check: type = dict):
"""Deserialize data from bridge
@param serialised_data(unicode): data to deserialise
@default (object): value to use when serialised data is empty string
@param type_check(type): if not None, the deserialised data must be of this type
@return(object): deserialised data
@raise ValueError: serialised_data is of wrong type
"""
if serialised_data == "":
return default
ret = json.loads(serialised_data)
if type_check is not None and not isinstance(ret, type_check):
raise ValueError("Bad data type, was expecting {type_check}, got {real_type}"
.format(type_check=type_check, real_type=type(ret)))
return ret

View file

@ -0,0 +1,213 @@
#!/usr/bin/env python3
# Libervia: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Objects handling bridge data, with jinja2 safe markup handling"""
from libervia.backend.core.constants import Const as C
from libervia.backend.tools.common import data_format
from os.path import basename
try:
from jinja2 import Markup as safe
except ImportError:
safe = str
from libervia.backend.tools.common import uri as xmpp_uri
import urllib.request, urllib.parse, urllib.error
q = lambda value: urllib.parse.quote(value.encode("utf-8"), safe="@")
class Message(object):
def __init__(self, msg_data):
self._uid = msg_data[0]
self._timestamp = msg_data[1]
self._from_jid = msg_data[2]
self._to_jid = msg_data[3]
self._message_data = msg_data[4]
self._subject_data = msg_data[5]
self._type = msg_data[6]
self._extra = data_format.deserialise(msg_data[7])
self._html = dict(data_format.get_sub_dict("xhtml", self._extra))
@property
def id(self):
return self._uid
@property
def timestamp(self):
return self._timestamp
@property
def from_(self):
return self._from_jid
@property
def text(self):
try:
return self._message_data[""]
except KeyError:
return next(iter(self._message_data.values()))
@property
def subject(self):
try:
return self._subject_data[""]
except KeyError:
return next(iter(self._subject_data.values()))
@property
def type(self):
return self._type
@property
def thread(self):
return self._extra.get("thread")
@property
def thread_parent(self):
return self._extra.get("thread_parent")
@property
def received(self):
return self._extra.get("received_timestamp", self._timestamp)
@property
def delay_sender(self):
return self._extra.get("delay_sender")
@property
def info_type(self):
return self._extra.get("info_type")
@property
def html(self):
if not self._html:
return None
try:
return safe(self._html[""])
except KeyError:
return safe(next(iter(self._html.values())))
class Messages(object):
def __init__(self, msgs_data):
self.messages = [Message(m) for m in msgs_data]
def __len__(self):
return self.messages.__len__()
def __missing__(self, key):
return self.messages.__missing__(key)
def __getitem__(self, key):
return self.messages.__getitem__(key)
def __iter__(self):
return self.messages.__iter__()
def __reversed__(self):
return self.messages.__reversed__()
def __contains__(self, item):
return self.messages.__contains__(item)
class Room(object):
def __init__(self, jid, name=None, url=None):
self.jid = jid
self.name = name or jid
if url is not None:
self.url = url
class Identity(object):
def __init__(self, jid_str, data=None):
self.jid_str = jid_str
self.data = data if data is not None else {}
@property
def avatar_basename(self):
try:
return basename(self.data['avatar']['path'])
except (TypeError, KeyError):
return None
def __getitem__(self, key):
return self.data[key]
def __getattr__(self, key):
try:
return self.data[key]
except KeyError:
raise AttributeError(key)
class Identities:
def __init__(self):
self.identities = {}
def __iter__(self):
return iter(self.identities)
def __getitem__(self, jid_str):
try:
return self.identities[jid_str]
except KeyError:
return None
def __setitem__(self, jid_str, data):
self.identities[jid_str] = Identity(jid_str, data)
def __contains__(self, jid_str):
return jid_str in self.identities
class ObjectQuoter(object):
"""object wrapper which quote attribues (to be used in templates)"""
def __init__(self, obj):
self.obj = obj
def __unicode__(self):
return q(self.obj.__unicode__())
def __str__(self):
return self.__unicode__()
def __getattr__(self, name):
return q(self.obj.__getattr__(name))
def __getitem__(self, key):
return q(self.obj.__getitem__(key))
class OnClick(object):
"""Class to handle clickable elements targets"""
def __init__(self, url=None):
self.url = url
def format_url(self, *args, **kwargs):
"""format URL using Python formatting
values will be quoted before being used
"""
return self.url.format(
*[q(a) for a in args], **{k: ObjectQuoter(v) for k, v in kwargs.items()}
)

View file

@ -0,0 +1,267 @@
#!/usr/bin/env python3
# SAT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""tools to help manipulating time and dates"""
from typing import Union
import calendar
import datetime
import re
import time
from babel import dates
from dateutil import parser, tz
from dateutil.parser import ParserError
from dateutil.relativedelta import relativedelta
from dateutil.utils import default_tzinfo
from libervia.backend.core import exceptions
from libervia.backend.core.constants import Const as C
from libervia.backend.core.i18n import _
RELATIVE_RE = re.compile(
r"\s*(?P<in>\bin\b)?"
r"(?P<date>[^+-].+[^\s+-])?\s*(?P<direction>[-+])?\s*"
r"\s*(?P<quantity>\d+)\s*"
r"(?P<unit>(second|sec|s|minute|min|month|mo|m|hour|hr|h|day|d|week|w|year|yr|y))s?"
r"(?P<ago>\s+ago)?\s*",
re.I
)
TIME_SYMBOL_MAP = {
"s": "second",
"sec": "second",
"m": "minute",
"min": "minute",
"h": "hour",
"hr": "hour",
"d": "day",
"w": "week",
"mo": "month",
"y": "year",
"yr": "year",
}
YEAR_FIRST_RE = re.compile(r"\d{4}[^\d]+")
TZ_UTC = tz.tzutc()
TZ_LOCAL = tz.gettz()
def date_parse(value, default_tz=TZ_UTC):
"""Parse a date and return corresponding unix timestamp
@param value(unicode): date to parse, in any format supported by parser
@param default_tz(datetime.tzinfo): default timezone
@return (int): timestamp
"""
value = str(value).strip()
dayfirst = False if YEAR_FIRST_RE.match(value) else True
try:
dt = default_tzinfo(
parser.parse(value, dayfirst=dayfirst),
default_tz)
except ParserError as e:
if value == "now":
dt = datetime.datetime.now(tz.tzutc())
else:
try:
# the date may already be a timestamp
return int(value)
except ValueError:
raise e
return calendar.timegm(dt.utctimetuple())
def date_parse_ext(value, default_tz=TZ_UTC):
"""Extended date parse which accept relative date
@param value(unicode): date to parse, in any format supported by parser
and with the hability to specify X days/weeks/months/years in the past or future.
Relative date are specified either with something like `[main_date] +1 week`
or with something like `3 days ago`, and it is case insensitive. [main_date] is
a date parsable by parser, or empty to specify current date/time.
"now" can also be used to specify current date/time.
@param default_tz(datetime.tzinfo): same as for date_parse
@return (int): timestamp
"""
m = RELATIVE_RE.match(value)
if m is None:
return date_parse(value, default_tz=default_tz)
if sum(1 for g in ("direction", "in", "ago") if m.group(g)) > 1:
raise ValueError(
_('You can use only one of direction (+ or -), "in" and "ago"'))
if m.group("direction") == '-' or m.group("ago"):
direction = -1
else:
direction = 1
date = m.group("date")
if date is not None:
date = date.strip()
if not date or date == "now":
dt = datetime.datetime.now(tz.tzutc())
else:
try:
dt = default_tzinfo(parser.parse(date, dayfirst=True), default_tz)
except ParserError as e:
try:
timestamp = int(date)
except ValueError:
raise e
else:
dt = datetime.datetime.fromtimestamp(timestamp, tz.tzutc())
quantity = int(m.group("quantity"))
unit = m.group("unit").lower()
try:
unit = TIME_SYMBOL_MAP[unit]
except KeyError:
pass
delta_kw = {f"{unit}s": direction * quantity}
dt = dt + relativedelta(**delta_kw)
return calendar.timegm(dt.utctimetuple())
def date_fmt(
timestamp: Union[float, int, str],
fmt: str = "short",
date_only: bool = False,
auto_limit: int = 7,
auto_old_fmt: str = "short",
auto_new_fmt: str = "relative",
locale_str: str = C.DEFAULT_LOCALE,
tz_info: Union[datetime.tzinfo, str] = TZ_UTC
) -> str:
"""Format date according to locale
@param timestamp: unix time
@param fmt: one of:
- short: e.g. u'31/12/17'
- medium: e.g. u'Apr 1, 2007'
- long: e.g. u'April 1, 2007'
- full: e.g. u'Sunday, April 1, 2007'
- relative: format in relative time
e.g.: 3 hours
note that this format is not precise
- iso: ISO 8601 format
e.g.: u'2007-04-01T19:53:23Z'
- auto: use auto_old_fmt if date is older than auto_limit
else use auto_new_fmt
- auto_day: shorcut to set auto format with change on day
old format will be short, and new format will be time only
or a free value which is passed to babel.dates.format_datetime
(see http://babel.pocoo.org/en/latest/dates.html?highlight=pattern#pattern-syntax)
@param date_only: if True, only display date (not datetime)
@param auto_limit: limit in days before using auto_old_fmt
use 0 to have a limit at last midnight (day change)
@param auto_old_fmt: format to use when date is older than limit
@param auto_new_fmt: format to use when date is equal to or more recent
than limit
@param locale_str: locale to use (as understood by babel)
@param tz_info: time zone to use
"""
timestamp = float(timestamp)
if isinstance(tz_info, str):
tz_info = tz.gettz(tz_info)
if fmt == "auto_day":
fmt, auto_limit, auto_old_fmt, auto_new_fmt = "auto", 0, "short", "HH:mm"
if fmt == "auto":
if auto_limit == 0:
now = datetime.datetime.now(tz_info)
# we want to use given tz_info, so we don't use date() or today()
today = datetime.datetime(year=now.year, month=now.month, day=now.day,
tzinfo=now.tzinfo)
today = calendar.timegm(today.utctimetuple())
if timestamp < today:
fmt = auto_old_fmt
else:
fmt = auto_new_fmt
else:
days_delta = (time.time() - timestamp) / 3600
if days_delta > (auto_limit or 7):
fmt = auto_old_fmt
else:
fmt = auto_new_fmt
if fmt == "relative":
delta = timestamp - time.time()
return dates.format_timedelta(
delta, granularity="minute", add_direction=True, locale=locale_str
)
elif fmt in ("short", "long", "full"):
if date_only:
dt = datetime.datetime.fromtimestamp(timestamp, tz_info)
return dates.format_date(dt, format=fmt, locale=locale_str)
else:
return dates.format_datetime(timestamp, format=fmt, locale=locale_str,
tzinfo=tz_info)
elif fmt == "iso":
if date_only:
fmt = "yyyy-MM-dd"
else:
fmt = "yyyy-MM-ddTHH:mm:ss'Z'"
return dates.format_datetime(timestamp, format=fmt)
else:
return dates.format_datetime(timestamp, format=fmt, locale=locale_str,
tzinfo=tz_info)
def delta2human(start_ts: Union[float, int], end_ts: Union[float, int]) -> str:
"""Convert delta of 2 unix times to human readable text
@param start_ts: timestamp of starting time
@param end_ts: timestamp of ending time
"""
if end_ts < start_ts:
raise exceptions.InternalError(
"end timestamp must be bigger or equal to start timestamp !"
)
rd = relativedelta(
datetime.datetime.fromtimestamp(end_ts),
datetime.datetime.fromtimestamp(start_ts)
)
text_elems = []
for unit in ("years", "months", "days", "hours", "minutes"):
value = getattr(rd, unit)
if value == 1:
# we remove final "s" when there is only 1
text_elems.append(f"1 {unit[:-1]}")
elif value > 1:
text_elems.append(f"{value} {unit}")
return ", ".join(text_elems)
def get_timezone_name(tzinfo, timestamp: Union[float, int]) -> str:
"""
Get the DST-aware timezone name for a given timezone and timestamp.
@param tzinfo: The timezone to get the name for
@param timestamp: The timestamp to use, as a Unix timestamp (number of seconds since
the Unix epoch).
@return: The DST-aware timezone name.
"""
dt = datetime.datetime.fromtimestamp(timestamp)
dt_tz = dt.replace(tzinfo=tzinfo)
tz_name = dt_tz.tzname()
if tz_name is None:
raise exceptions.InternalError("tz_name should not be None")
return tz_name

View file

@ -0,0 +1,43 @@
#!/usr/bin/env python3
# Libervia: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
""" tools dynamic import """
from importlib import import_module
from libervia.backend.core.log import getLogger
log = getLogger(__name__)
def bridge(name, module_path="libervia.backend.bridge"):
"""import bridge module
@param module_path(str): path of the module to import
@param name(str): name of the bridge to import (e.g.: dbus)
@return (module, None): imported module or None if nothing is found
"""
try:
bridge_module = import_module(module_path + "." + name)
except ImportError:
try:
bridge_module = import_module(module_path + "." + name + "_bridge")
except ImportError as e:
log.warning(f"Can't import bridge {name!r}: {e}")
bridge_module = None
return bridge_module

View file

@ -0,0 +1,84 @@
#!/usr/bin/env python3
# SàT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""email sending facilities"""
from email.mime.text import MIMEText
from twisted.mail import smtp
from libervia.backend.core.constants import Const as C
from libervia.backend.tools import config as tools_config
from libervia.backend.core.log import getLogger
log = getLogger(__name__)
def send_email(config, to_emails, subject="", body="", from_email=None):
"""Send an email using SàT configuration
@param config (SafeConfigParser): the configuration instance
@param to_emails(list[unicode], unicode): list of recipients
if unicode, it will be split to get emails
@param subject(unicode): subject of the message
@param body(unicode): body of the message
@param from_email(unicode): address of the sender
@return (D): same as smtp.sendmail
"""
if isinstance(to_emails, str):
to_emails = to_emails.split()
email_host = tools_config.config_get(config, None, "email_server") or "localhost"
email_from = from_email or tools_config.config_get(config, None, "email_from")
# we suppose that email domain and XMPP domain are identical
domain = tools_config.config_get(config, None, "xmpp_domain")
if domain is None:
if email_from is not None:
domain = email_from.split("@", 1)[-1]
else:
domain = "example.net"
if email_from is None:
email_from = "no_reply@" + domain
email_sender_domain = tools_config.config_get(
config, None, "email_sender_domain", domain
)
email_port = int(tools_config.config_get(config, None, "email_port", 25))
email_username = tools_config.config_get(config, None, "email_username")
email_password = tools_config.config_get(config, None, "email_password")
email_auth = C.bool(tools_config.config_get(config, None, "email_auth", C.BOOL_FALSE))
email_starttls = C.bool(tools_config.config_get(config, None, "email_starttls",
C.BOOL_FALSE))
msg = MIMEText(body, "plain", "UTF-8")
msg["Subject"] = subject
msg["From"] = email_from
msg["To"] = ", ".join(to_emails)
return smtp.sendmail(
email_host.encode("utf-8"),
email_from.encode("utf-8"),
[email.encode("utf-8") for email in to_emails],
msg.as_bytes(),
senderDomainName=email_sender_domain if email_sender_domain else None,
port=email_port,
username=email_username.encode("utf-8") if email_username else None,
password=email_password.encode("utf-8") if email_password else None,
requireAuthentication=email_auth,
requireTransportSecurity=email_starttls,
)

View file

@ -0,0 +1,34 @@
#!/usr/bin/env python3
# SaT: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""tools to help manipulating files"""
from pathlib import Path
def get_unique_name(path):
"""Generate a path with a name not conflicting with existing file
@param path(str, Path): path to the file to create
@return (Path): unique path (can be the same as path if there is no conflict)
"""
ori_path = path = Path(path)
idx = 1
while path.exists():
path = ori_path.with_name(f"{ori_path.stem}_{idx}{ori_path.suffix}")
idx += 1
return path

View file

@ -0,0 +1,95 @@
#!/usr/bin/env python3
# Salut à Toi: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
""" regex tools common to backend and frontends """
import re
import unicodedata
path_escape = {"%": "%25", "/": "%2F", "\\": "%5c"}
path_escape_rev = {re.escape(v): k for k, v in path_escape.items()}
path_escape = {re.escape(k): v for k, v in path_escape.items()}
#  thanks to Martijn Pieters (https://stackoverflow.com/a/14693789)
RE_ANSI_REMOVE = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
RE_TEXT_URL = re.compile(r'[^a-zA-Z0-9,_]+')
TEXT_MAX_LEN = 60
# min lenght is currently deactivated
TEXT_WORD_MIN_LENGHT = 0
def re_join(exps):
"""Join (OR) various regexes"""
return re.compile("|".join(exps))
def re_sub_dict(pattern, repl_dict, string):
"""Replace key, value found in dict according to pattern
@param pattern(basestr): pattern using keys found in repl_dict
@repl_dict(dict): keys found in this dict will be replaced by
corresponding values
@param string(basestr): string to use for the replacement
"""
return pattern.sub(lambda m: repl_dict[re.escape(m.group(0))], string)
path_escape_re = re_join(list(path_escape.keys()))
path_escape_rev_re = re_join(list(path_escape_rev.keys()))
def path_escape(string):
"""Escape string so it can be use in a file path
@param string(basestr): string to escape
@return (str, unicode): escaped string, usable in a file path
"""
return re_sub_dict(path_escape_re, path_escape, string)
def path_unescape(string):
"""Unescape string from value found in file path
@param string(basestr): string found in file path
@return (str, unicode): unescaped string
"""
return re_sub_dict(path_escape_rev_re, path_escape_rev, string)
def ansi_remove(string):
"""Remove ANSI escape codes from string
@param string(basestr): string to filter
@return (str, unicode): string without ANSI escape codes
"""
return RE_ANSI_REMOVE.sub("", string)
def url_friendly_text(text):
"""Convert text to url-friendly one"""
# we change special chars to ascii one,
# trick found at https://stackoverflow.com/a/3194567
text = unicodedata.normalize('NFD', text).encode('ascii', 'ignore').decode('utf-8')
text = RE_TEXT_URL.sub(' ', text).lower()
text = '-'.join([t for t in text.split() if t and len(t)>=TEXT_WORD_MIN_LENGHT])
while len(text) > TEXT_MAX_LEN:
if '-' in text:
text = text.rsplit('-', 1)[0]
else:
text = text[:TEXT_MAX_LEN]
return text

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,243 @@
#!/usr/bin/env python3
# SAT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
""" template XMLUI parsing
XMLUI classes from this modules can then be iterated to create the template
"""
from functools import partial
from libervia.backend.core.log import getLogger
from sat_frontends.tools import xmlui
from sat_frontends.tools import jid
try:
from jinja2 import Markup as safe
except ImportError:
# Safe marks XHTML values as usable as it.
# If jinja2 is not there, we can use a simple lamba
safe = lambda x: x
log = getLogger(__name__)
## Widgets ##
class Widget(object):
category = "widget"
type = None
enabled = True
read_only = True
def __init__(self, xmlui_parent):
self.xmlui_parent = xmlui_parent
@property
def name(self):
return self._xmlui_name
class ValueWidget(Widget):
def __init__(self, xmlui_parent, value):
super(ValueWidget, self).__init__(xmlui_parent)
self.value = value
@property
def values(self):
return [self.value]
@property
def labels(self):
#  helper property, there is not label for ValueWidget
# but using labels make rendering more easy (one single method to call)
#  values are actually returned
return [self.value]
class InputWidget(ValueWidget):
def __init__(self, xmlui_parent, value, read_only=False):
super(InputWidget, self).__init__(xmlui_parent, value)
self.read_only = read_only
class OptionsWidget(Widget):
def __init__(self, xmlui_parent, options, selected, style):
super(OptionsWidget, self).__init__(xmlui_parent)
self.options = options
self.selected = selected
self.style = style
@property
def values(self):
for value, label in self.items:
yield value
@property
def value(self):
if self.multi or self.no_select or len(self.selected) != 1:
raise ValueError(
"Can't get value for list with multiple selections or nothing selected"
)
return self.selected[0]
@property
def labels(self):
"""return only labels from self.items"""
for value, label in self.items:
yield label
@property
def items(self):
"""return suitable items, according to style"""
no_select = self.no_select
for value, label in self.options:
if no_select or value in self.selected:
yield value, label
@property
def inline(self):
return "inline" in self.style
@property
def no_select(self):
return "noselect" in self.style
@property
def multi(self):
return "multi" in self.style
class EmptyWidget(xmlui.EmptyWidget, Widget):
def __init__(self, _xmlui_parent):
Widget.__init__(self)
class TextWidget(xmlui.TextWidget, ValueWidget):
type = "text"
class JidWidget(xmlui.JidWidget, ValueWidget):
type = "jid"
def __init__(self, xmlui_parent, value):
self.value = jid.JID(value)
class LabelWidget(xmlui.LabelWidget, ValueWidget):
type = "label"
@property
def for_name(self):
try:
return self._xmlui_for_name
except AttributeError:
return None
class StringWidget(xmlui.StringWidget, InputWidget):
type = "string"
class JidInputWidget(xmlui.JidInputWidget, StringWidget):
type = "jid"
class TextBoxWidget(xmlui.TextWidget, InputWidget):
type = "textbox"
class XHTMLBoxWidget(xmlui.XHTMLBoxWidget, InputWidget):
type = "xhtmlbox"
def __init__(self, xmlui_parent, value, read_only=False):
# XXX: XHTMLBoxWidget value must be cleaned (harmful XHTML must be removed)
# This is normally done in the backend, the frontends should not need to
# worry about it.
super(XHTMLBoxWidget, self).__init__(
xmlui_parent=xmlui_parent, value=safe(value), read_only=read_only)
class ListWidget(xmlui.ListWidget, OptionsWidget):
type = "list"
## Containers ##
class Container(object):
category = "container"
type = None
def __init__(self, xmlui_parent):
self.xmlui_parent = xmlui_parent
self.children = []
def __iter__(self):
return iter(self.children)
def _xmlui_append(self, widget):
self.children.append(widget)
def _xmlui_remove(self, widget):
self.children.remove(widget)
class VerticalContainer(xmlui.VerticalContainer, Container):
type = "vertical"
class PairsContainer(xmlui.PairsContainer, Container):
type = "pairs"
class LabelContainer(xmlui.PairsContainer, Container):
type = "label"
## Factory ##
class WidgetFactory(object):
def __getattr__(self, attr):
if attr.startswith("create"):
cls = globals()[attr[6:]]
return cls
## Core ##
class XMLUIPanel(xmlui.XMLUIPanel):
widget_factory = WidgetFactory()
def show(self, *args, **kwargs):
raise NotImplementedError
class XMLUIDialog(xmlui.XMLUIDialog):
dialog_factory = WidgetFactory()
def __init__(*args, **kwargs):
raise NotImplementedError
create = partial(xmlui.create, class_map={
xmlui.CLASS_PANEL: XMLUIPanel,
xmlui.CLASS_DIALOG: XMLUIDialog})

View file

@ -0,0 +1,140 @@
#!/usr/bin/env python3
# Libervia: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""TLS handling with twisted"""
from libervia.backend.core.log import getLogger
from libervia.backend.core import exceptions
from libervia.backend.tools import config as tools_config
try:
import OpenSSL
from twisted.internet import ssl
except ImportError:
ssl = None
log = getLogger(__name__)
def get_options_from_config(config, section=""):
options = {}
for option in ('tls_certificate', 'tls_private_key', 'tls_chain'):
options[option] = tools_config.config_get(config, section, option)
return options
def tls_options_check(options):
"""Check options coherence if TLS is activated, and update missing values
Must be called only if TLS is activated
"""
if not options["tls_certificate"]:
raise exceptions.ConfigError(
"a TLS certificate is needed to activate HTTPS connection")
if not options["tls_private_key"]:
options["tls_private_key"] = options["tls_certificate"]
def load_certificates(f):
"""Read a .pem file with a list of certificates
@param f (file): file obj (opened .pem file)
@return (list[OpenSSL.crypto.X509]): list of certificates
@raise OpenSSL.crypto.Error: error while parsing the file
"""
# XXX: didn't found any method to load a .pem file with several certificates
# so the certificates split is done here
certificates = []
buf = []
while True:
line = f.readline()
buf.append(line)
if "-----END CERTIFICATE-----" in line:
certificates.append(
OpenSSL.crypto.load_certificate(
OpenSSL.crypto.FILETYPE_PEM, "".join(buf)
)
)
buf = []
elif not line:
log.debug(f"{len(certificates)} certificate(s) found")
return certificates
def load_p_key(f):
"""Read a private key from a .pem file
@param f (file): file obj (opened .pem file)
@return (list[OpenSSL.crypto.PKey]): private key object
@raise OpenSSL.crypto.Error: error while parsing the file
"""
return OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, f.read())
def load_certificate(f):
"""Read a public certificate from a .pem file
@param f (file): file obj (opened .pem file)
@return (list[OpenSSL.crypto.X509]): public certificate
@raise OpenSSL.crypto.Error: error while parsing the file
"""
return OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, f.read())
def get_tls_context_factory(options):
"""Load TLS certificate and build the context factory needed for listenSSL"""
if ssl is None:
raise ImportError("Python module pyOpenSSL is not installed!")
cert_options = {}
for name, option, method in [
("privateKey", "tls_private_key", load_p_key),
("certificate", "tls_certificate", load_certificate),
("extraCertChain", "tls_chain", load_certificates),
]:
path = options[option]
if not path:
assert option == "tls_chain"
continue
log.debug(f"loading {option} from {path}")
try:
with open(path) as f:
cert_options[name] = method(f)
except IOError as e:
raise exceptions.DataError(
f"Error while reading file {path} for option {option}: {e}"
)
except OpenSSL.crypto.Error:
raise exceptions.DataError(
f"Error while parsing file {path} for option {option}, are you sure "
f"it is a valid .pem file?"
)
if (
option == "tls_private_key"
and options["tls_certificate"] == path
):
raise exceptions.ConfigError(
f"You are using the same file for private key and public "
f"certificate, make sure that both a in {path} or use "
f"--tls_private_key option"
)
return ssl.CertificateOptions(**cert_options)

View file

@ -0,0 +1,121 @@
#!/usr/bin/env python3
# SAT: a jabber client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
""" XMPP uri parsing tools """
from typing import Optional
import sys
import urllib.parse
import urllib.request, urllib.parse, urllib.error
# FIXME: basic implementation, need to follow RFC 5122
def parse_xmpp_uri(uri):
"""Parse an XMPP uri and return a dict with various information
@param uri(unicode): uri to parse
@return dict(unicode, unicode): data depending of the URI where key can be:
type: one of ("pubsub", TODO)
type is always present
sub_type: can be:
- microblog
only used for pubsub for now
path: XMPP path (jid of the service or entity)
node: node used
id: id of the element (item for pubsub)
@raise ValueError: the scheme is not xmpp
"""
uri_split = urllib.parse.urlsplit(uri)
if uri_split.scheme != "xmpp":
raise ValueError("this is not a XMPP URI")
# XXX: we don't use jid.JID for path as it can be used both in backend and frontend
# which may use different JID classes
data = {"path": urllib.parse.unquote(uri_split.path)}
query_end = uri_split.query.find(";")
if query_end == -1:
# we just have a JID
query_type = None
else:
query_type = uri_split.query[:query_end]
if "=" in query_type:
raise ValueError("no query type, invalid XMPP URI")
if sys.version_info >= (3, 9):
# parse_qs behaviour has been modified in Python 3.9, ";" is not understood as a
# parameter separator anymore but the "separator" argument has been added to
# change it.
pairs = urllib.parse.parse_qs(uri_split.geturl(), separator=";")
else:
pairs = urllib.parse.parse_qs(uri_split.geturl())
for k, v in list(pairs.items()):
if len(v) != 1:
raise NotImplementedError("multiple values not managed")
if k in ("path", "type", "sub_type"):
raise NotImplementedError("reserved key used in URI, this is not supported")
data[k] = urllib.parse.unquote(v[0])
if query_type:
data["type"] = query_type
elif "node" in data:
data["type"] = "pubsub"
else:
data["type"] = ""
if "node" in data:
if data["node"].startswith("urn:xmpp:microblog:"):
data["sub_type"] = "microblog"
return data
def add_pairs(uri, pairs):
for k, v in pairs.items():
uri.append(
";"
+ urllib.parse.quote_plus(k.encode("utf-8"))
+ "="
+ urllib.parse.quote_plus(v.encode("utf-8"))
)
def build_xmpp_uri(type_: Optional[str] = None, **kwargs: str) -> str:
uri = ["xmpp:"]
subtype = kwargs.pop("subtype", None)
path = kwargs.pop("path")
uri.append(urllib.parse.quote_plus(path.encode("utf-8")).replace("%40", "@"))
if type_ is None:
# we have an URI to a JID
if kwargs:
raise NotImplementedError(
"keyword arguments are not supported for URI without type"
)
elif type_ == "pubsub":
if subtype == "microblog" and not kwargs.get("node"):
kwargs["node"] = "urn:xmpp:microblog:0"
if kwargs:
uri.append("?")
add_pairs(uri, kwargs)
else:
raise NotImplementedError("{type_} URI are not handled yet".format(type_=type_))
return "".join(uri)

View file

@ -0,0 +1,159 @@
#!/usr/bin/env python3
# Libervia: an XMPP client
# Copyright (C) 2009-2021 Jérôme Poisson (goffi@goffi.org)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Misc utils for both backend and frontends"""
import collections.abc
size_units = {
"b": 1,
"kb": 1000,
"mb": 1000**2,
"gb": 1000**3,
"tb": 1000**4,
"pb": 1000**5,
"eb": 1000**6,
"zb": 1000**7,
"yb": 1000**8,
"o": 1,
"ko": 1000,
"mo": 1000**2,
"go": 1000**3,
"to": 1000**4,
"po": 1000**5,
"eo": 1000**6,
"zo": 1000**7,
"yo": 1000**8,
"kib": 1024,
"mib": 1024**2,
"gib": 1024**3,
"tib": 1024**4,
"pib": 1024**5,
"eib": 1024**6,
"zib": 1024**7,
"yib": 1024**8,
"kio": 1024,
"mio": 1024**2,
"gio": 1024**3,
"tio": 1024**4,
"pio": 1024**5,
"eio": 1024**6,
"zio": 1024**7,
"yio": 1024**8,
}
def per_luminance(red, green, blue):
"""Caculate the perceived luminance of a RGB color
@param red(int): 0-1 normalized value of red
@param green(int): 0-1 normalized value of green
@param blue(int): 0-1 normalized value of blue
@return (float): 0-1 value of luminance (<0.5 is dark, else it's light)
"""
# cf. https://stackoverflow.com/a/1855903, thanks Gacek
return 0.299 * red + 0.587 * green + 0.114 * blue
def recursive_update(ori: dict, update: dict):
"""Recursively update a dictionary"""
# cf. https://stackoverflow.com/a/3233356, thanks Alex Martelli
for k, v in update.items():
if isinstance(v, collections.abc.Mapping):
ori[k] = recursive_update(ori.get(k, {}), v)
else:
ori[k] = v
return ori
class OrderedSet(collections.abc.MutableSet):
"""A mutable sequence which doesn't keep duplicates"""
# TODO: complete missing set methods
def __init__(self, values=None):
self._dict = {}
if values is not None:
self.update(values)
def __len__(self):
return len(self._dict)
def __iter__(self):
return iter(self._dict.keys())
def __contains__(self, item):
return item in self._dict
def add(self, item):
self._dict[item] = None
def discard(self, item):
try:
del self._dict[item]
except KeyError:
pass
def update(self, items):
self._dict.update({i: None for i in items})
def parse_size(size):
"""Parse a file size with optional multiple symbole"""
try:
return int(size)
except ValueError:
number = []
symbol = []
try:
for c in size:
if c == " ":
continue
if c.isdigit():
number.append(c)
elif c.isalpha():
symbol.append(c)
else:
raise ValueError("unexpected char in size: {c!r} (size: {size!r})")
number = int("".join(number))
symbol = "".join(symbol)
if symbol:
try:
multiplier = size_units[symbol.lower()]
except KeyError:
raise ValueError(
"unknown size multiplier symbole: {symbol!r} (size: {size!r})")
else:
return number * multiplier
return number
except Exception as e:
raise ValueError(f"invalid size: {e}")
def get_size_multiplier(size, suffix="o"):
"""Get multiplier of a file size"""
size = int(size)
#  cf. https://stackoverflow.com/a/1094933 (thanks)
for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
if abs(size) < 1024.0:
return size, f"{unit}{suffix}"
size /= 1024.0
return size, f"Yi{suffix}"
def get_human_size(size, suffix="o", sep=" "):
size, symbol = get_size_multiplier(size, suffix)
return f"{size:.2f}{sep}{symbol}"