code style reformatting using black
This commit is contained in:
Goffi
parent
4127bcffd0
commit
744cb0c503
1 changed files with 21394 additions and 10488 deletions
|
|
@ -20,6 +20,7 @@
|
|||
from sat.core.i18n import _
|
||||
from sat.core.constants import Const as C
|
||||
from sat.core.log import getLogger
|
||||
|
||||
log = getLogger(__name__)
|
||||
from twisted.internet import defer
|
||||
from sat.core import exceptions
|
||||
|
|
@ -37,14 +38,13 @@ PLUGIN_INFO = {
|
|||
C.PI_DEPENDENCIES: [],
|
||||
C.PI_MAIN: "ImportPlugin",
|
||||
C.PI_HANDLER: "no",
|
||||
C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers""")
|
||||
C.PI_DESCRIPTION: _(u"""Generic import plugin, base for specialized importers"""),
|
||||
}
|
||||
|
||||
Importer = collections.namedtuple('Importer', ('callback', 'short_desc', 'long_desc'))
|
||||
Importer = collections.namedtuple("Importer", ("callback", "short_desc", "long_desc"))
|
||||
|
||||
|
||||
class ImportPlugin(object):
|
||||
|
||||
def __init__(self, host):
|
||||
log.info(_("plugin Import initialization"))
|
||||
self.host = host
|
||||
|
|
@ -64,21 +64,51 @@ class ImportPlugin(object):
|
|||
@param name(unicode): import handler name
|
||||
"""
|
||||
assert name == name.lower().strip()
|
||||
log.info(_(u'initializing {name} import handler').format(name=name))
|
||||
log.info(_(u"initializing {name} import handler").format(name=name))
|
||||
import_handler.name = name
|
||||
import_handler.register = partial(self.register, import_handler)
|
||||
import_handler.unregister = partial(self.unregister, import_handler)
|
||||
import_handler.importers = {}
|
||||
|
||||
def _import(name, location, options, pubsub_service, pubsub_node, profile):
|
||||
return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile)
|
||||
return self._doImport(
|
||||
import_handler,
|
||||
name,
|
||||
location,
|
||||
options,
|
||||
pubsub_service,
|
||||
pubsub_node,
|
||||
profile,
|
||||
)
|
||||
|
||||
def _importList():
|
||||
return self.listImporters(import_handler)
|
||||
|
||||
def _importDesc(name):
|
||||
return self.getDescription(import_handler, name)
|
||||
|
||||
self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True)
|
||||
self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList)
|
||||
self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc)
|
||||
self.host.bridge.addMethod(
|
||||
name + "Import",
|
||||
".plugin",
|
||||
in_sign="ssa{ss}sss",
|
||||
out_sign="s",
|
||||
method=_import,
|
||||
async=True,
|
||||
)
|
||||
self.host.bridge.addMethod(
|
||||
name + "ImportList",
|
||||
".plugin",
|
||||
in_sign="",
|
||||
out_sign="a(ss)",
|
||||
method=_importList,
|
||||
)
|
||||
self.host.bridge.addMethod(
|
||||
name + "ImportDesc",
|
||||
".plugin",
|
||||
in_sign="s",
|
||||
out_sign="(ss)",
|
||||
method=_importDesc,
|
||||
)
|
||||
|
||||
def getProgress(self, import_handler, progress_id, profile):
|
||||
client = self.host.getClient(profile)
|
||||
|
|
@ -87,7 +117,10 @@ class ImportPlugin(object):
|
|||
def listImporters(self, import_handler):
|
||||
importers = import_handler.importers.keys()
|
||||
importers.sort()
|
||||
return [(name, import_handler.importers[name].short_desc) for name in import_handler.importers]
|
||||
return [
|
||||
(name, import_handler.importers[name].short_desc)
|
||||
for name in import_handler.importers
|
||||
]
|
||||
|
||||
def getDescription(self, import_handler, name):
|
||||
"""Return import short and long descriptions
|
||||
|
|
@ -98,13 +131,24 @@ class ImportPlugin(object):
|
|||
try:
|
||||
importer = import_handler.importers[name]
|
||||
except KeyError:
|
||||
raise exceptions.NotFound(u"{handler_name} importer not found [{name}]".format(
|
||||
handler_name = import_handler.name,
|
||||
name = name))
|
||||
raise exceptions.NotFound(
|
||||
u"{handler_name} importer not found [{name}]".format(
|
||||
handler_name=import_handler.name, name=name
|
||||
)
|
||||
)
|
||||
else:
|
||||
return importer.short_desc, importer.long_desc
|
||||
|
||||
def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE):
|
||||
def _doImport(
|
||||
self,
|
||||
import_handler,
|
||||
name,
|
||||
location,
|
||||
options,
|
||||
pubsub_service="",
|
||||
pubsub_node="",
|
||||
profile=C.PROF_KEY_NONE,
|
||||
):
|
||||
client = self.host.getClient(profile)
|
||||
options = {key: unicode(value) for key, value in options.iteritems()}
|
||||
for option in import_handler.BOOL_OPTIONS:
|
||||
|
|
@ -116,12 +160,31 @@ class ImportPlugin(object):
|
|||
try:
|
||||
options[option] = json.loads(options[option])
|
||||
except ValueError:
|
||||
raise exceptions.DataError(_(u'invalid json option: {name}').format(name=option))
|
||||
raise exceptions.DataError(
|
||||
_(u"invalid json option: {name}").format(name=option)
|
||||
)
|
||||
pubsub_service = jid.JID(pubsub_service) if pubsub_service else None
|
||||
return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service, pubsub_node or None)
|
||||
return self.doImport(
|
||||
client,
|
||||
import_handler,
|
||||
unicode(name),
|
||||
unicode(location),
|
||||
options,
|
||||
pubsub_service,
|
||||
pubsub_node or None,
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None):
|
||||
def doImport(
|
||||
self,
|
||||
client,
|
||||
import_handler,
|
||||
name,
|
||||
location,
|
||||
options=None,
|
||||
pubsub_service=None,
|
||||
pubsub_node=None,
|
||||
):
|
||||
"""Import data
|
||||
|
||||
@param import_handler(object): instance of the import handler
|
||||
|
|
@ -142,7 +205,7 @@ class ImportPlugin(object):
|
|||
for opt_name, opt_default in import_handler.OPT_DEFAULTS.iteritems():
|
||||
# we want a filled options dict, with all empty or False values removed
|
||||
try:
|
||||
value =options[opt_name]
|
||||
value = options[opt_name]
|
||||
except KeyError:
|
||||
if opt_default:
|
||||
options[opt_name] = opt_default
|
||||
|
|
@ -154,31 +217,61 @@ class ImportPlugin(object):
|
|||
importer = import_handler.importers[name]
|
||||
except KeyError:
|
||||
raise exceptions.NotFound(u"Importer [{}] not found".format(name))
|
||||
items_import_data, items_count = yield importer.callback(client, location, options)
|
||||
items_import_data, items_count = yield importer.callback(
|
||||
client, location, options
|
||||
)
|
||||
progress_id = unicode(uuid.uuid4())
|
||||
try:
|
||||
_import = client._import
|
||||
except AttributeError:
|
||||
_import = client._import = {}
|
||||
progress_data = _import.setdefault(import_handler.name, {})
|
||||
progress_data[progress_id] = {u'position': '0'}
|
||||
progress_data[progress_id] = {u"position": "0"}
|
||||
if items_count is not None:
|
||||
progress_data[progress_id]['size'] = unicode(items_count)
|
||||
metadata = {'name': u'{}: {}'.format(name, location),
|
||||
'direction': 'out',
|
||||
'type': import_handler.name.upper() + '_IMPORT'
|
||||
}
|
||||
self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile)
|
||||
self.host.bridge.progressStarted(progress_id, metadata, client.profile)
|
||||
session = { # session data, can be used by importers
|
||||
u'root_service': pubsub_service,
|
||||
u'root_node': pubsub_node
|
||||
progress_data[progress_id]["size"] = unicode(items_count)
|
||||
metadata = {
|
||||
"name": u"{}: {}".format(name, location),
|
||||
"direction": "out",
|
||||
"type": import_handler.name.upper() + "_IMPORT",
|
||||
}
|
||||
self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node)
|
||||
self.host.registerProgressCb(
|
||||
progress_id,
|
||||
partial(self.getProgress, import_handler),
|
||||
metadata,
|
||||
profile=client.profile,
|
||||
)
|
||||
self.host.bridge.progressStarted(progress_id, metadata, client.profile)
|
||||
session = { # session data, can be used by importers
|
||||
u"root_service": pubsub_service,
|
||||
u"root_node": pubsub_node,
|
||||
}
|
||||
self.recursiveImport(
|
||||
client,
|
||||
import_handler,
|
||||
items_import_data,
|
||||
progress_id,
|
||||
session,
|
||||
options,
|
||||
None,
|
||||
pubsub_service,
|
||||
pubsub_node,
|
||||
)
|
||||
defer.returnValue(progress_id)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0):
|
||||
def recursiveImport(
|
||||
self,
|
||||
client,
|
||||
import_handler,
|
||||
items_import_data,
|
||||
progress_id,
|
||||
session,
|
||||
options,
|
||||
return_data=None,
|
||||
service=None,
|
||||
node=None,
|
||||
depth=0,
|
||||
):
|
||||
"""Do the import recursively
|
||||
|
||||
@param import_handler(object): instance of the import handler
|
||||
|
|
@ -196,33 +289,37 @@ class ImportPlugin(object):
|
|||
if return_data is None:
|
||||
return_data = {}
|
||||
for idx, item_import_data in enumerate(items_import_data):
|
||||
item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node)
|
||||
item_data = yield import_handler.importItem(
|
||||
client, item_import_data, session, options, return_data, service, node
|
||||
)
|
||||
yield import_handler.itemFilters(client, item_data, session, options)
|
||||
recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options)
|
||||
recurse_kwargs = yield import_handler.importSubItems(
|
||||
client, item_import_data, item_data, session, options
|
||||
)
|
||||
yield import_handler.publishItem(client, item_data, service, node, session)
|
||||
|
||||
if recurse_kwargs is not None:
|
||||
recurse_kwargs['client'] = client
|
||||
recurse_kwargs['import_handler'] = import_handler
|
||||
recurse_kwargs['progress_id'] = progress_id
|
||||
recurse_kwargs['session'] = session
|
||||
recurse_kwargs.setdefault('options', options)
|
||||
recurse_kwargs['return_data'] = return_data
|
||||
recurse_kwargs['depth'] = depth + 1
|
||||
recurse_kwargs["client"] = client
|
||||
recurse_kwargs["import_handler"] = import_handler
|
||||
recurse_kwargs["progress_id"] = progress_id
|
||||
recurse_kwargs["session"] = session
|
||||
recurse_kwargs.setdefault("options", options)
|
||||
recurse_kwargs["return_data"] = return_data
|
||||
recurse_kwargs["depth"] = depth + 1
|
||||
log.debug(_(u"uploading subitems"))
|
||||
yield self.recursiveImport(**recurse_kwargs)
|
||||
|
||||
if depth == 0:
|
||||
client._import[import_handler.name][progress_id]['position'] = unicode(idx+1)
|
||||
client._import[import_handler.name][progress_id]["position"] = unicode(
|
||||
idx + 1
|
||||
)
|
||||
|
||||
if depth == 0:
|
||||
self.host.bridge.progressFinished(progress_id,
|
||||
return_data,
|
||||
client.profile)
|
||||
self.host.bridge.progressFinished(progress_id, return_data, client.profile)
|
||||
self.host.removeProgressCb(progress_id, client.profile)
|
||||
del client._import[import_handler.name][progress_id]
|
||||
|
||||
def register(self, import_handler, name, callback, short_desc='', long_desc=''):
|
||||
def register(self, import_handler, name, callback, short_desc="", long_desc=""):
|
||||
"""Register an Importer method
|
||||
|
||||
@param name(unicode): unique importer name, should indicate the software it can import and always lowercase
|
||||
|
|
@ -239,9 +336,11 @@ class ImportPlugin(object):
|
|||
"""
|
||||
name = name.lower()
|
||||
if name in import_handler.importers:
|
||||
raise exceptions.ConflictError(_(u"An {handler_name} importer with the name {name} already exist").format(
|
||||
handler_name = import_handler.name,
|
||||
name = name))
|
||||
raise exceptions.ConflictError(
|
||||
_(
|
||||
u"An {handler_name} importer with the name {name} already exist"
|
||||
).format(handler_name=import_handler.name, name=name)
|
||||
)
|
||||
import_handler.importers[name] = Importer(callback, short_desc, long_desc)
|
||||
|
||||
def unregister(self, import_handler, name):
|
||||
|
|
|
|||
Loading…
Reference in a new issue