memory (persistent, sqlite): better private values handling + new LazyPersistentBinaryDict:
- merged private values method handling in sqlite, and added a keys arguments to get only some keys
- LazyPersistentBinaryDict allow to get values in database only when they are needed, saving memory for big data
This commit is contained in:
Goffi
parent
25c07ce105
commit
ceab0c6bdc
1 changed files with 194 additions and 187 deletions
|
|
@ -28,30 +28,38 @@ class MemoryNotInitializedError(Exception):
|
|||
|
||||
class PersistentDict(object):
|
||||
r"""A dictionary which save persistently each value assigned
|
||||
|
||||
/!\ be careful, each assignment means a database write
|
||||
/!\ Memory must be initialised before loading/setting value with instances of this class"""
|
||||
storage = None
|
||||
binary = False
|
||||
|
||||
def __init__(self, namespace, profile=None):
|
||||
"""@param namespace: unique namespace for this dictionary
|
||||
@param profile: profile which *MUST* exists, or None for general values"""
|
||||
"""
|
||||
|
||||
@param namespace: unique namespace for this dictionary
|
||||
@param profile(unicode, None): profile which *MUST* exists, or None for general values
|
||||
"""
|
||||
if not self.storage:
|
||||
log.error(_("PersistentDict can't be used before memory initialisation"))
|
||||
raise MemoryNotInitializedError
|
||||
self._cache = {}
|
||||
self._cache = None
|
||||
self.namespace = namespace
|
||||
self.profile = profile
|
||||
|
||||
def _setCache(self, data):
|
||||
self._cache = data
|
||||
|
||||
def load(self):
|
||||
"""Load persistent data from storage.
|
||||
|
||||
need to be called before any other operation
|
||||
@return: defers the PersistentDict instance itself
|
||||
"""
|
||||
if not self.profile:
|
||||
d = self.storage.loadGenPrivates(self._cache, self.namespace)
|
||||
else:
|
||||
d = self.storage.loadIndPrivates(self._cache, self.namespace, self.profile)
|
||||
return d.addCallback(lambda dummy: self)
|
||||
d = self.storage.getPrivates(self.namespace, binary=self.binary, profile=self.profile)
|
||||
d.addCallback(self._setCache)
|
||||
d.addCallback(lambda dummy: self)
|
||||
return d
|
||||
|
||||
def iteritems(self):
|
||||
return self._cache.iteritems()
|
||||
|
|
@ -102,17 +110,11 @@ class PersistentDict(object):
|
|||
return self._cache.__getitem__(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
if not self.profile:
|
||||
self.storage.setGenPrivate(self.namespace, key, value)
|
||||
else:
|
||||
self.storage.setIndPrivate(self.namespace, key, value, self.profile)
|
||||
self.storage.setPrivateValue(self.namespace, key, value, self.binary, self.profile)
|
||||
return self._cache.__setitem__(key, value)
|
||||
|
||||
def __delitem__(self, key):
|
||||
if not self.profile:
|
||||
self.storage.delGenPrivate(self.namespace, key)
|
||||
else:
|
||||
self.storage.delIndPrivate(self.namespace, key, self.profile)
|
||||
self.storage.detPrivateValue(self.namepace, key, self.binary, self.profile)
|
||||
return self._cache.__delitem__(key)
|
||||
|
||||
def get(self, key, default=None):
|
||||
|
|
@ -127,40 +129,103 @@ class PersistentDict(object):
|
|||
|
||||
def force(self, name):
|
||||
"""Force saving of an attribute to storage
|
||||
@return: deferred fired when data is actually saved"""
|
||||
if not self.profile:
|
||||
return self.storage.setGenPrivate(self.namespace, name, self._cache[name])
|
||||
return self.storage.setIndPrivate(self.namespace, name, self._cache[name], self.profile)
|
||||
|
||||
@return: deferred fired when data is actually saved
|
||||
"""
|
||||
return self.storage.setPrivateValue(self.namespace, name, self._cache[name], self.binary, self.profile)
|
||||
|
||||
|
||||
class PersistentBinaryDict(PersistentDict):
|
||||
"""Persistent dict where value can be any python data (instead of string only)"""
|
||||
binary = True
|
||||
|
||||
|
||||
class LazyPersistentBinaryDict(PersistentBinaryDict):
|
||||
ur"""PersistentBinaryDict which get key/value when needed
|
||||
|
||||
This Persistent need more database access, it is suitable for largest data,
|
||||
to save memory.
|
||||
/!\ most of methods return a Deferred
|
||||
"""
|
||||
# TODO: missing methods should be implemented using database access
|
||||
# TODO: a cache would be useful (which is deleted after a timeout)
|
||||
|
||||
def load(self):
|
||||
"""load persistent data from storage
|
||||
"""
|
||||
if not self.profile:
|
||||
return self.storage.loadGenPrivatesBinary(self._cache, self.namespace)
|
||||
else:
|
||||
return self.storage.loadIndPrivatesBinary(self._cache, self.namespace, self.profile)
|
||||
# we show a warning as calling load on LazyPersistentBinaryDict sounds like a code mistake
|
||||
log.warning(_(u"Calling load on LazyPersistentBinaryDict while it's not needed"))
|
||||
|
||||
def iteritems(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def items(self):
|
||||
return self.storage.getPrivates(self.namespace, binary=self.binary, profile=self.profile)
|
||||
|
||||
def __repr__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def __str__(self):
|
||||
return "LazyPersistentBinaryDict (namespace: {})".format(self.namespace)
|
||||
|
||||
def __lt__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __le__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __eq__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __ne__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __gt__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __ge__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __cmp__(self, other):
|
||||
raise NotImplementedError
|
||||
|
||||
def __hash__(self):
|
||||
return hash(unicode(self.__class__) + self.namespace + (self.profile or u''))
|
||||
|
||||
def __nonzero__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def __contains__(self, key):
|
||||
raise NotImplementedError
|
||||
|
||||
def __iter__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""get the value as a Deferred"""
|
||||
return self.storage.getPrivates(self.namespace, keys=[key], binary=self.binary, profile=self.profile)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
if not self.profile:
|
||||
self.storage.setGenPrivateBinary(self.namespace, key, value)
|
||||
else:
|
||||
self.storage.setIndPrivateBinary(self.namespace, key, value, self.profile)
|
||||
return self._cache.__setitem__(key, value)
|
||||
self.storage.setPrivateValue(self.namespace, key, value, self.binary, self.profile)
|
||||
|
||||
def __delitem__(self, key):
|
||||
if not self.profile:
|
||||
self.storage.delGenPrivateBinary(self.namespace, key)
|
||||
else:
|
||||
self.storage.delIndPrivateBinary(self.namespace, key, self.profile)
|
||||
return self._cache.__delitem__(key)
|
||||
self.storage.detPrivateValue(self.namepace, key, self.binary, self.profile)
|
||||
|
||||
def force(self, name):
|
||||
def _valueOrDefault(self, value, default):
|
||||
if value is None:
|
||||
return default
|
||||
return value
|
||||
|
||||
def get(self, key, default=None):
|
||||
d = self.__getitem__(key)
|
||||
d.addCallback(self._valueOrDefault)
|
||||
return d
|
||||
|
||||
def setdefault(self, key, default):
|
||||
raise NotImplementedError
|
||||
|
||||
def force(self, name, value):
|
||||
"""Force saving of an attribute to storage
|
||||
@return: deferred fired when data is actually saved"""
|
||||
if not self.profile:
|
||||
return self.storage.setGenPrivateBinary(self.namespace, name, self._cache[name])
|
||||
return self.storage.setIndPrivateBinary(self.namespace, name, self._cache[name], self.profile)
|
||||
|
||||
@param value(object): value is needed for LazyPersistentBinaryDict
|
||||
@return: deferred fired when data is actually saved
|
||||
"""
|
||||
return self.storage.setPrivateValue(self.namespace, name, value, self.binary, self.profile)
|
||||
|
|
|
|||
Loading…
Reference in a new issue