erebion/libervia-backend
Watch
1
0
Fork
You've already forked libervia-backend
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

plugin import: node can now be specified + added a "session" dict to keep import session data:

Browse source 
import session data are data that can be used by importer to store anything which can be useful to keep between import methods.
This commit is contained in:
Goffi 2017-10-06 08:52:51 +02:00
commit 1bcdca4a2f
2 changed files with 26 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/plugins/plugin_blog_import.py
View file

@ -27,7 +27,6 @@ from twisted.web import client as web_client
from twisted.words.xish import domish
from sat.core import exceptions
from sat.tools import xml_tools
import collections
import os
import os.path
import tempfile
@ -53,9 +52,6 @@ OPT_IGNORE_TLS = 'ignore_tls_errors'
URL_REDIRECT_PREFIX = 'url_redirect_'
BlogImporter = collections.namedtuple('BlogImporter', ('callback', 'short_desc', 'long_desc'))
class BlogImportPlugin(object):
BOOL_OPTIONS = (OPT_UPLOAD_IMAGES, OPT_IGNORE_TLS)
OPT_DEFAULTS = {OPT_UPLOAD_IMAGES: True,
@ -70,10 +66,10 @@ class BlogImportPlugin(object):
self._s = self.host.plugins['TEXT-SYNTAXES']
host.plugins['IMPORT'].initialize(self, u'blog')
def importItem(self, client, item_import_data, options, return_data, service, node):
def importItem(self, client, item_import_data, session, options, return_data, service, node):
"""importItem specialized for blog import
@param items_import_data(iterable[dict]):
@param item_import_data(dict):
* mandatory keys:
'blog' (dict): microblog data of the blog post (cf. http://wiki.goffi.org/wiki/Bridge_API_-_Microblogging/en)
the importer MUST NOT create node or call XEP-0277 plugin itself
@ -124,7 +120,7 @@ class BlogImportPlugin(object):
return mb_data
def importSubItems(self, client, item_import_data, mb_data, options):
def importSubItems(self, client, item_import_data, mb_data, session, options):
# comments data
if len(item_import_data['comments']) != 1:
raise NotImplementedError(u"can't manage multiple comment links")
@ -143,12 +139,12 @@ class BlogImportPlugin(object):
raise exceptions.DataError(u"allow_comments set to False, but comments are there")
return None
def publishItem(self, client, mb_data, service, node):
def publishItem(self, client, mb_data, service, node, session):
log.debug(u"uploading item [{id}]: {title}".format(id=mb_data['id'], title=mb_data.get('title','')))
return self._m.send(client, mb_data, service, node)
@defer.inlineCallbacks
def itemFilters(self, client, mb_data, options):
def itemFilters(self, client, mb_data, session, options):
"""Apply filters according to options
modify mb_data in place

31
src/plugins/plugin_import.py
View file

@ -65,14 +65,14 @@ class ImportPlugin(object):
import_handler.register = partial(self.register, import_handler)
import_handler.unregister = partial(self.unregister, import_handler)
import_handler.importers = {}
def _import(name, location, options, pubsub_service, profile):
return self._doImport(import_handler, name, location, options, pubsub_service, profile)
def _import(name, location, options, pubsub_service, pubsub_node, profile):
return self._doImport(import_handler, name, location, options, pubsub_service, pubsub_node, profile)
def _importList():
return self.listImporters(import_handler)
def _importDesc(name):
return self.getDescription(import_handler, name)
self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}ss', out_sign='s', method=_import, async=True)
self.host.bridge.addMethod(name + "Import", ".plugin", in_sign='ssa{ss}sss', out_sign='s', method=_import, async=True)
self.host.bridge.addMethod(name + "ImportList", ".plugin", in_sign='', out_sign='a(ss)', method=_importList)
self.host.bridge.addMethod(name + "ImportDesc", ".plugin", in_sign='s', out_sign='(ss)', method=_importDesc)
@ -100,7 +100,7 @@ class ImportPlugin(object):
else:
return importer.short_desc, importer.long_desc
def _doImport(self, import_handler, name, location, options, pubsub_service='', profile=C.PROF_KEY_NONE):
def _doImport(self, import_handler, name, location, options, pubsub_service='', pubsub_node='', profile=C.PROF_KEY_NONE):
client = self.host.getClient(profile)
options = {key: unicode(value) for key, value in options.iteritems()}
for option in import_handler.BOOL_OPTIONS:
@ -108,10 +108,10 @@ class ImportPlugin(object):
options[option] = C.bool(options[option])
except KeyError:
pass
return self.doImport(client, import_handler, unicode(name), unicode(location), options)
return self.doImport(client, import_handler, unicode(name), unicode(location), options, pubsub_service or None, pubsub_node or None)
@defer.inlineCallbacks
def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None):
def doImport(self, client, import_handler, name, location, options=None, pubsub_service=None, pubsub_node=None):
"""Import data
@param import_handler(object): instance of the import handler
@ -122,6 +122,8 @@ class ImportPlugin(object):
@param options(dict, None): extra options.
@param pubsub_service(jid.JID, None): jid of the PubSub service where data must be imported
None to use profile's server
@param pubsub_node(unicode, None): PubSub node to use
None to use importer's default node
@return (unicode): progress id
"""
if options is None:
@ -157,17 +159,19 @@ class ImportPlugin(object):
}
self.host.registerProgressCb(progress_id, partial(self.getProgress, import_handler), metadata, profile=client.profile)
self.host.bridge.progressStarted(progress_id, metadata, client.profile)
url_redirect = {}
self.recursiveImport(client, import_handler, items_import_data, progress_id, options, url_redirect)
session = {} # session data, can be use by importers
self.recursiveImport(client, import_handler, items_import_data, progress_id, session, options, None, pubsub_service, pubsub_node)
defer.returnValue(progress_id)
@defer.inlineCallbacks
def recursiveImport(self, client, import_handler, items_import_data, progress_id, options, return_data=None, service=None, node=None, depth=0):
def recursiveImport(self, client, import_handler, items_import_data, progress_id, session, options, return_data=None, service=None, node=None, depth=0):
"""Do the import recursively
@param import_handler(object): instance of the import handler
@param items_import_data(iterable): iterable of data as specified in [register]
@param progress_id(unicode): id of progression
@param session(dict): data for this import session
can be used by importer so store any useful data
@param options(dict): import options
@param return_data(dict): data to return on progressFinished
@param service(jid.JID, None): PubSub service to use
@ -177,15 +181,16 @@ class ImportPlugin(object):
if return_data is None:
return_data = {}
for idx, item_import_data in enumerate(items_import_data):
item_data = yield import_handler.importItem(client, item_import_data, options, return_data, service, node)
yield import_handler.itemFilters(client, item_data, options)
recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, options)
yield import_handler.publishItem(client, item_data, service, node)
item_data = yield import_handler.importItem(client, item_import_data, session, options, return_data, service, node)
yield import_handler.itemFilters(client, item_data, session, options)
recurse_kwargs = yield import_handler.importSubItems(client, item_import_data, item_data, session, options)
yield import_handler.publishItem(client, item_data, service, node, session)
if recurse_kwargs is not None:
recurse_kwargs['client'] = client
recurse_kwargs['import_handler'] = import_handler
recurse_kwargs['progress_id'] = progress_id
recurse_kwargs['session'] = session
recurse_kwargs.setdefault('options', options)
recurse_kwargs['return_data'] = return_data
recurse_kwargs['depth'] = depth + 1