diff --git a/external_file_location/README.rst b/external_file_location/README.rst new file mode 100644 index 00000000000..4f39f1707e5 --- /dev/null +++ b/external_file_location/README.rst @@ -0,0 +1,60 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License + +External File Location +====================== + +This module was written to extend the functionality of ir.attachment to support remote communication and allow you to import/export file to a remote server + +Installation +============ + +To install this module, you need to: + +* fs python module +* Paramiko python module + +Usage +===== + +To use this module, you need to: + +* Add a location with your server infos +* Create a task with your file info and remote communication method +* A cron task will trigger each task + +For further information, please visit: + +* https://www.odoo.com/forum/help-1 + +Known issues / Roadmap +====================== + + +Credits +======= + +* Joel Grand-Guillaume Camptocamp +* initOS +* Valentin CHEMIERE +* Mourad EL HADJ MIMOUNE + + +Contributors +------------ + +* Sebastien BEAU +* David BEAL + +Maintainer +---------- + +.. image:: http://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: http://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. + +To contribute to this module, please visit http://odoo-community.org. diff --git a/external_file_location/__init__.py b/external_file_location/__init__.py new file mode 100644 index 00000000000..47f1a67776d --- /dev/null +++ b/external_file_location/__init__.py @@ -0,0 +1,3 @@ +from . import models +from . import tasks +from . import tests diff --git a/external_file_location/__openerp__.py b/external_file_location/__openerp__.py new file mode 100644 index 00000000000..e544ab7333f --- /dev/null +++ b/external_file_location/__openerp__.py @@ -0,0 +1,33 @@ +# coding: utf-8 +# @ 2015 Valentin CHEMIERE @ Akretion +# © 2016 @author Mourad EL HADJ MIMOUNE +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +{ + 'name': 'external_file_location', + 'version': '8.0.1.0.0', + 'author': 'Akretion,Odoo Community Association (OCA)', + 'website': 'www.akretion.com', + 'license': 'AGPL-3', + 'category': 'Generic Modules', + 'depends': [ + 'attachment_metadata', + ], + 'external_dependencies': { + 'python': [ + 'fs', + 'paramiko', + ], + }, + 'data': [ + 'views/menu.xml', + 'views/attachment_view.xml', + 'views/location_view.xml', + 'views/task_view.xml', + 'data/cron.xml', + 'security/ir.model.access.csv', + ], + 'installable': True, + 'application': False, + 'images': [], +} diff --git a/external_file_location/data/cron.xml b/external_file_location/data/cron.xml new file mode 100644 index 00000000000..a18f5f6a2dc --- /dev/null +++ b/external_file_location/data/cron.xml @@ -0,0 +1,18 @@ + + + + + + Run file exchange tasks + 30 + minutes + -1 + True + + external.file.task + _run + ([]) + + + + diff --git a/external_file_location/models/__init__.py b/external_file_location/models/__init__.py new file mode 100644 index 00000000000..6f41762054a --- /dev/null +++ b/external_file_location/models/__init__.py @@ -0,0 +1,3 @@ +from . import attachment +from . import location +from . import task diff --git a/external_file_location/models/attachment.py b/external_file_location/models/attachment.py new file mode 100644 index 00000000000..49424bc4337 --- /dev/null +++ b/external_file_location/models/attachment.py @@ -0,0 +1,21 @@ +# coding: utf-8 +# @ 2015 Valentin CHEMIERE @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from openerp import models, fields + + +class IrAttachmentMetadata(models.Model): + _inherit = 'ir.attachment.metadata' + + sync_date = fields.Datetime() + state = fields.Selection([ + ('pending', 'Pending'), + ('failed', 'Failed'), + ('done', 'Done'), + ], readonly=False, required=True, default='pending') + state_message = fields.Text() + task_id = fields.Many2one('external.file.task', string='Task') + location_id = fields.Many2one( + 'external.file.location', string='Location', + related='task_id.location_id', store=True) diff --git a/external_file_location/models/helper.py b/external_file_location/models/helper.py new file mode 100644 index 00000000000..f19302033a5 --- /dev/null +++ b/external_file_location/models/helper.py @@ -0,0 +1,84 @@ +# coding: utf-8 +# Author: Joel Grand-Guillaume +# Copyright 2011-2012 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + + +def itersubclasses(cls, _seen=None): + """ + itersubclasses(cls) + Generator over all subclasses of a given class, in depth first order. + >>> list(itersubclasses(int)) == [bool] + True + >>> class A(object): pass + >>> class B(A): pass + >>> class C(A): pass + >>> class D(B,C): pass + >>> class E(D): pass + >>> + >>> for cls in itersubclasses(A): + ... print(cls.__name__) + B + D + E + C + >>> # get ALL (new-style) classes currently defined + >>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS + ['type', ...'tuple', ...] + """ + if not isinstance(cls, type): + raise TypeError('itersubclasses must be called with ' + 'new-style classes, not %.100r' % cls + ) + if _seen is None: + _seen = set() + try: + subs = cls.__subclasses__() + except TypeError: # fails only when cls is type + subs = cls.__subclasses__(cls) + for sub in subs: + if sub not in _seen: + _seen.add(sub) + yield sub + for sub in itersubclasses(sub, _seen): + yield sub + + +def _get_erp_module_name(module_path): + # see this PR for v9 https://github.com/odoo/odoo/pull/11084 + """ Extract the name of the Odoo module from the path of the + Python module. + + Taken from Odoo server: ``openerp.models.MetaModel`` + + The (Odoo) module name can be in the ``openerp.addons`` namespace + or not. For instance module ``sale`` can be imported as + ``openerp.addons.sale`` (the good way) or ``sale`` (for backward + compatibility). + """ + module_parts = module_path.split('.') + if len(module_parts) > 2 and module_parts[:2] == ['openerp', 'addons']: + module_name = module_parts[2] + else: + module_name = module_parts[0] + return module_name + + +def is_module_installed(env, module_name): + """ Check if an Odoo addon is installed. + + :param module_name: name of the addon + """ + # the registry maintains a set of fully loaded modules so we can + # lookup for our module there + return module_name in env.registry._init_modules + + +def get_erp_module(cls_or_func): + """ For a top level function or class, returns the + name of the Odoo module where it lives. + + So we will be able to filter them according to the modules + installation state. + """ + return _get_erp_module_name(cls_or_func.__module__) diff --git a/external_file_location/models/location.py b/external_file_location/models/location.py new file mode 100644 index 00000000000..8a3d800cbf9 --- /dev/null +++ b/external_file_location/models/location.py @@ -0,0 +1,51 @@ +# coding: utf-8 +# @ 2015 Valentin CHEMIERE @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from openerp import models, fields, api +from ..tasks.abstract_task import AbstractTask +from .helper import itersubclasses + + +class Location(models.Model): + _name = 'external.file.location' + _description = 'Description' + + name = fields.Char(string='Name', required=True) + protocol = fields.Selection(selection='_get_protocol', required=True) + address = fields.Char(string='Address', required=True) + port = fields.Integer() + login = fields.Char() + password = fields.Char() + task_ids = fields.One2many('external.file.task', 'location_id') + hide_login = fields.Boolean() + hide_password = fields.Boolean() + hide_port = fields.Boolean() + + def _get_protocol(self): + res = [] + for cls in itersubclasses(AbstractTask): + if not cls._synchronize_type: + cls_info = (cls._key, cls._name) + res.append(cls_info) + elif not cls._synchronize_type and cls._key and cls._name: + pass + return res + + @api.onchange('protocol') + def onchange_protocol(self): + for cls in itersubclasses(AbstractTask): + if cls._key == self.protocol: + self.port = cls._default_port + if cls._hide_login: + self.hide_login = True + else: + self.hide_login = False + if cls._hide_password: + self.hide_password = True + else: + self.hide_password = False + if cls._hide_port: + self.hide_port = True + else: + self.hide_port = False diff --git a/external_file_location/models/task.py b/external_file_location/models/task.py new file mode 100644 index 00000000000..64e9478f04b --- /dev/null +++ b/external_file_location/models/task.py @@ -0,0 +1,115 @@ +# coding: utf-8 +# @ 2015 Valentin CHEMIERE @ Akretion +# © @author Mourad EL HADJ MIMOUNE +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from openerp import models, fields, api +from .helper import itersubclasses, get_erp_module, is_module_installed +from ..tasks.abstract_task import AbstractTask + + +class Task(models.Model): + _name = 'external.file.task' + _description = 'External file task' + + name = fields.Char(required=True) + method = fields.Selection(selection='_get_method', required=True, + help='procotol and trasmitting info') + method_type = fields.Char() + filename = fields.Char(help='File name which is imported.' + 'You can use file pattern like *.txt' + 'to import all txt files') + filepath = fields.Char(help='Path to imported file') + location_id = fields.Many2one('external.file.location', string='Location', + required=True) + attachment_ids = fields.One2many('ir.attachment.metadata', 'task_id', + string='Attachment') + move_path = fields.Char(string='Move path', + help='Imported File will be moved to this path') + new_name = fields.Char(string='New name', + help='Imported File will be renamed to this name' + 'Name can use mako template where obj is an ' + 'ir_attachement. template exemple : ' + ' ${obj.name}-${obj.create_date}.csv') + md5_check = fields.Boolean(help='Control file integrity after import with' + ' a md5 file') + after_import = fields.Selection(selection='_get_action', + help='Action after import a file') + file_type = fields.Selection( + selection="_get_file_type", + string="File type", + help="The file type determines an import method to be used " + "to parse and transform data before their import in ERP") + + def _get_action(self): + return [('rename', 'Rename'), + ('move', 'Move'), + ('move_rename', 'Move & Rename'), + ('delete', 'Delete'), + ] + + def _get_file_type(self): + """This is the method to be inherited for adding file types + The basic import do not apply any parsing or transform of the file. + The file is just added as an attachement + """ + return [('basic_import', 'Basic import')] + + def _get_method(self): + res = [] + for cls in itersubclasses(AbstractTask): + if not is_module_installed(self.env, get_erp_module(cls)): + continue + if cls._synchronize_type and ( + 'protocol' not in self._context or + cls._key == self._context['protocol']): + cls_info = (cls._key + '_' + cls._synchronize_type, + cls._name + ' ' + cls._synchronize_type) + res.append(cls_info) + return res + + @api.onchange('method') + def onchange_method(self): + if self.method: + if 'import' in self.method: + self.method_type = 'import' + elif 'export' in self.method: + self.method_type = 'export' + + @api.model + def _run(self, domain=None): + if not domain: + domain = [] + tasks = self.env['external.file.task'].search(domain) + tasks.run() + + @api.multi + def run(self): + for tsk in self: + for cls in itersubclasses(AbstractTask): + if not is_module_installed(self.env, get_erp_module(cls)): + continue + cls_build = '%s_%s' % (cls._key, cls._synchronize_type) + if cls._synchronize_type and cls_build == tsk.method: + method_class = cls + config = { + 'host': tsk.location_id.address, + # ftplib does not support unicode + 'user': tsk.location_id.login and\ + tsk.location_id.login.encode('utf-8'), + 'pwd': tsk.location_id.password and \ + tsk.location_id.password.encode('utf-8'), + 'port': tsk.location_id.port, + 'allow_dir_creation': False, + 'file_name': tsk.filename, + 'path': tsk.filepath, + 'attachment_ids': tsk.attachment_ids, + 'task': tsk, + 'move_path': tsk.move_path, + 'new_name': tsk.new_name, + 'after_import': tsk.after_import, + 'file_type': tsk.file_type, + 'md5_check': tsk.md5_check, + } + conn = method_class(self.env, config) + conn.run() diff --git a/external_file_location/security/ir.model.access.csv b/external_file_location/security/ir.model.access.csv new file mode 100644 index 00000000000..37961f1954f --- /dev/null +++ b/external_file_location/security/ir.model.access.csv @@ -0,0 +1,5 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_external_file_location_manager,external.file.location.manager,model_external_file_location,base.group_system,1,1,1,1 +access_external_file_location_user,external.file.location.user,model_external_file_location,base.group_user,1,0,0,0 +access_external_file_task_manager,external.file.task.manager,model_external_file_task,base.group_system,1,1,1,1 +access_external_file_task_user,external.file.task.user,model_external_file_task,base.group_user,1,0,0,0 diff --git a/external_file_location/tasks/__init__.py b/external_file_location/tasks/__init__.py new file mode 100644 index 00000000000..49ace2611fb --- /dev/null +++ b/external_file_location/tasks/__init__.py @@ -0,0 +1,4 @@ +from . import abstract_fs +from . import ftp +from . import sftp +from . import filestore diff --git a/external_file_location/tasks/abstract_fs.py b/external_file_location/tasks/abstract_fs.py new file mode 100644 index 00000000000..03539f8d96e --- /dev/null +++ b/external_file_location/tasks/abstract_fs.py @@ -0,0 +1,196 @@ +# coding: utf-8 +# Copyright (C) 2014 initOS GmbH & Co. KG (). +# @ 2015 Valentin CHEMIERE @ Akretion +# ©2016 @author Mourad EL HADJ MIMOUNE +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import logging +import os +import fnmatch +import datetime + +from openerp import tools + +from .abstract_task import AbstractTask + +_logger = logging.getLogger(__name__) + + +try: + # We use a jinja2 sandboxed environment to render mako templates. + # Note that the rendering does not cover all the mako syntax, in particular + # arbitrary Python statements are not accepted, and not all expressions are + # allowed: only "public" attributes (not starting with '_') of objects may + # be accessed. + # This is done on purpose: it prevents incidental or malicious execution of + # Python code that may break the security of the server. + from jinja2.sandbox import SandboxedEnvironment + mako_template_env = SandboxedEnvironment( + variable_start_string="${", + variable_end_string="}", + line_statement_prefix="%", + trim_blocks=True, # do not output newline after blocks + ) + mako_template_env.globals.update({ + 'str': str, + 'datetime': datetime, + 'len': len, + 'abs': abs, + 'min': min, + 'max': max, + 'sum': sum, + 'filter': filter, + 'reduce': reduce, + 'map': map, + 'round': round, + }) +except ImportError: + _logger.warning("jinja2 not available, templating features will not work!") + + +class AbstractFSTask(AbstractTask): + + _name = None + _key = None + _synchronize_type = None + _default_port = None + + def __init__(self, env, config): + self.env = env + self.host = config.get('host', '') + self.user = config.get('user', '') + self.pwd = config.get('pwd', '') + self.port = config.get('port', '') + self.allow_dir_creation = config.get('allow_dir_creation', '') + self.file_name = config.get('file_name', '') + self.path = config.get('path') or '.' + self.move_path = config.get('move_path', '') + self.new_name = config.get('new_name', '') + self.after_import = config.get('after_import', False) + self.file_type = config.get('file_type', False) + self.attachment_ids = config.get('attachment_ids', False) + self.task = config.get('task', False) + self.ext_hash = False + self.md5_check = config.get('md5_check', False) + + def _handle_new_source(self, fs_conn, download_directory, file_name, + move_directory): + """open and read given file into create_file method, + move file if move_directory is given""" + with fs_conn.open(self._source_name(download_directory, file_name), + "rb") as fileobj: + data = fileobj.read() + return self.create_file(file_name, data) + + def _source_name(self, download_directory, file_name): + """helper to get the full name""" + return os.path.join(download_directory, file_name) + + def _move_file(self, fs_conn, source, target): + """Moves a file on the server""" + _logger.info('Moving file %s %s' % (source, target)) + fs_conn.rename(source, target) + if self.md5_check: + fs_conn.rename(source + '.md5', target + '.md5') + + def _delete_file(self, fs_conn, source): + """Deletes a file from the server""" + _logger.info('Deleting file %s' % source) + fs_conn.remove(source) + if self.md5_check: + fs_conn.remove(source + '.md5') + + def _get_hash(self, file_name, fs_conn): + hash_file_name = file_name + '.md5' + with fs_conn.open(hash_file_name, 'rb') as f: + return f.read().rstrip('\r\n') + + def _get_files(self, conn, path): + process_files = [] + files_list = conn.listdir(path) + pattern = self.file_name + for file_name in fnmatch.filter(files_list, pattern): + source_name = self._source_name(self.path, file_name) + process_files.append((file_name, source_name)) + return process_files + + def _template_render(self, template, record): + try: + template = mako_template_env.from_string(tools.ustr(template)) + except Exception: + _logger.exception("Failed to load template %r", template) + + variables = {'obj': record} + try: + render_result = template.render(variables) + except Exception: + _logger.exception( + "Failed to render template %r using values %r" % + (template, variables)) + render_result = u"" + if render_result == u"False": + render_result = u"" + return render_result + + def _process_file(self, conn, file_to_process): + if self.md5_check: + self.ext_hash = self._get_hash(file_to_process[1], conn) + att_id = self._handle_new_source( + conn, + self.path, + self.file_name, + self.move_path) + move = False + rename = False + if self.after_import: + move = 'move' in self.after_import + rename = 'rename' in self.after_import + + # Move/rename/delete files only after all + # files have been processed. + if self.after_import == 'delete': + self._delete_file(conn, file_to_process[1]) + elif rename or move: + new_name = file_to_process[0] + if rename and self.new_name: + new_name_render = self._template_render( + self.new_name, att_id) + if new_name_render: + # Avoid space in file name + new_name = new_name_render.replace(' ', '_') + if self.move_path and not conn.exists(self.move_path): + conn.makedir(self.move_path) + move_path = self.move_path if self.move_path else self.path + self._move_file( + conn, + file_to_process[1], + self._source_name(move_path, new_name)) + return att_id + + def _handle_existing_target(self, fs_conn, target_name, filedata): + raise Exception("%s already exists" % target_name) + + def _handle_new_target(self, fs_conn, target_name, filedata): + try: + with fs_conn.open(target_name, mode='wb') as fileobj: + fileobj.write(filedata) + _logger.info('wrote %s, size %d', target_name, len(filedata)) + self.attachment_id.state = 'done' + self.attachment_id.state_message = '' + except IOError: + self.attachment_id.state = 'failed' + self.attachment_id.state_message = ( + 'The directory doesn\'t exist or had insufficient rights') + + def _target_name(self, fs_conn, upload_directory, filename): + return os.path.join(upload_directory, filename) + + def _upload_file(self, conn, host, port, user, pwd, + path, filename, filedata): + upload_directory = path or '.' + target_name = self._target_name(conn, + upload_directory, + filename) + if conn.isfile(target_name): + self._handle_existing_target(conn, target_name, filedata) + else: + self._handle_new_target(conn, target_name, filedata) diff --git a/external_file_location/tasks/abstract_task.py b/external_file_location/tasks/abstract_task.py new file mode 100644 index 00000000000..c40a6e9b781 --- /dev/null +++ b/external_file_location/tasks/abstract_task.py @@ -0,0 +1,28 @@ +# coding: utf-8 +# @ 2015 Valentin CHEMIERE @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from base64 import b64encode + + +class AbstractTask(object): + + _name = None + _key = None + _synchronize_type = None + _default_port = None + _hide_login = False + _hide_password = False + _hide_port = False + + def create_file(self, filename, data): + ir_attachment_id = self.env['ir.attachment.metadata'].create({ + 'name': filename, + 'datas': b64encode(data), + 'datas_fname': filename, + 'task_id': self.task and self.task.id or False, + 'location_id': self.task and self.task.location_id.id or False, + 'external_hash': self.ext_hash, + 'file_type': self.file_type, + }) + return ir_attachment_id diff --git a/external_file_location/tasks/filestore.py b/external_file_location/tasks/filestore.py new file mode 100644 index 00000000000..24e51e78367 --- /dev/null +++ b/external_file_location/tasks/filestore.py @@ -0,0 +1,53 @@ +# coding: utf-8 +# Copyright (C) 2014 initOS GmbH & Co. KG (). +# @ 2015 Valentin CHEMIERE @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from .abstract_fs import AbstractFSTask +from base64 import b64decode +from fs import osfs +import logging +_logger = logging.getLogger(__name__) + + +class FileStoreTask(AbstractFSTask): + + _key = 'filestore' + _name = 'File Store' + _synchronize_type = None + _default_port = None + _hide_login = True + _hide_password = True + _hide_port = True + + +class FileStoreImportTask(FileStoreTask): + + _synchronize_type = 'import' + + def run(self): + att_ids = [] + with osfs.OSFS(self.host) as fs_conn: + files_to_process = self._get_files(fs_conn, self.path) + for file_to_process in files_to_process: + att_ids.append(self._process_file(fs_conn, file_to_process)) + return att_ids + + +class FileStoreExportTask(FileStoreTask): + + _synchronize_type = 'export' + + def run(self, async=True): + for attachment in self.attachment_ids: + if attachment.state in ('pending', 'failed'): + self.attachment_id = attachment + with osfs.OSFS(self.host) as fs_conn: + self._upload_file(fs_conn, + self.host, + self.port, + self.user, + self.pwd, + self.path, + attachment.datas_fname, + b64decode(attachment.datas)) diff --git a/external_file_location/tasks/ftp.py b/external_file_location/tasks/ftp.py new file mode 100644 index 00000000000..aa01778d5aa --- /dev/null +++ b/external_file_location/tasks/ftp.py @@ -0,0 +1,51 @@ +# coding: utf-8 +# Copyright (C) 2014 initOS GmbH & Co. KG (). +# @ 2015 Valentin CHEMIERE @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from .abstract_fs import AbstractFSTask +from base64 import b64decode +from fs import ftpfs +import logging +_logger = logging.getLogger(__name__) + + +class FtpTask(AbstractFSTask): + + _key = 'ftp' + _name = 'FTP' + _synchronize_type = None + _default_port = 21 + _hide_login = False + _hide_password = False + _hide_port = False + + +class FtpImportTask(FtpTask): + + _synchronize_type = 'import' + + def run(self): + att_ids = [] + with ftpfs.FTPFS( + self.host, self.user, self.pwd, port=self.port) as ftp_conn: + files_to_process = self._get_files(ftp_conn, self.path) + for file_to_process in files_to_process: + att_ids.append(self._process_file(ftp_conn, file_to_process)) + return att_ids + + +class FtpExportTask(FtpTask): + + _synchronize_type = 'export' + + def run(self, async=True): + for attachment in self.attachment_ids: + if attachment.state in ('pending', 'failed'): + self.attachment_id = attachment + with ftpfs.FTPFS(self.host, self.user, self.pwd, + port=self.port) as ftp_conn: + self._upload_file(ftp_conn, self.host, self.port, + self.user, self.pwd, self.path, + attachment.datas_fname, + b64decode(attachment.datas)) diff --git a/external_file_location/tasks/sftp.py b/external_file_location/tasks/sftp.py new file mode 100644 index 00000000000..6d4f6988a65 --- /dev/null +++ b/external_file_location/tasks/sftp.py @@ -0,0 +1,57 @@ +# coding: utf-8 +# Copyright (C) 2014 initOS GmbH & Co. KG (). +# @ 2015 Valentin CHEMIERE @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from .abstract_fs import AbstractFSTask +from base64 import b64decode +from fs import sftpfs +import logging +_logger = logging.getLogger(__name__) + + +class SftpTask(AbstractFSTask): + + _key = 'sftp' + _name = 'SFTP' + _synchronize_type = None + _default_port = 22 + _hide_login = False + _hide_password = False + _hide_port = False + + +class SftpImportTask(SftpTask): + + _synchronize_type = 'import' + + def run(self): + connection_string = "{}:{}".format(self.host, self.port) + root = "/" + att_ids = [] + with sftpfs.SFTPFS(connection=connection_string, + root_path=root, + username=self.user, + password=self.pwd) as sftp_conn: + files_to_process = self._get_files(sftp_conn, self.path) + for file_to_process in files_to_process: + att_ids.append(self._process_file(sftp_conn, file_to_process)) + return att_ids + + +class SftpExportTask(SftpTask): + + _synchronize_type = 'export' + + def run(self, async=True): + for attachment in self.attachment_ids: + if attachment.state in ('pending', 'failed'): + self.attachment_id = attachment + connection_string = "{}:{}".format(self.host, self.port) + with sftpfs.SFTPFS(connection=connection_string, + username=self.user, + password=self.pwd) as sftp_conn: + datas = b64decode(attachment.datas) + self._upload_file(sftp_conn, self.host, self.port, + self.user, self.pwd, self.path, + attachment.datas_fname, datas) diff --git a/external_file_location/tests/__init__.py b/external_file_location/tests/__init__.py new file mode 100644 index 00000000000..af79fd981ab --- /dev/null +++ b/external_file_location/tests/__init__.py @@ -0,0 +1,2 @@ +from . import mock_server +from . import test_sftp diff --git a/external_file_location/tests/mock_server.py b/external_file_location/tests/mock_server.py new file mode 100644 index 00000000000..5589b739925 --- /dev/null +++ b/external_file_location/tests/mock_server.py @@ -0,0 +1,59 @@ +# coding: utf-8 +# Copyright (C) 2014 initOS GmbH & Co. KG (). +# @ 2015 Valentin CHEMIERE @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import mock +from contextlib import contextmanager +from collections import defaultdict + + +class MultiResponse(dict): + pass + + +class ConnMock(object): + + def __init__(self, response): + self.response = response + self._calls = [] + self.call_count = defaultdict(int) + + def __getattribute__(self, method): + if method not in ('_calls', 'response', 'call_count'): + def callable(*args, **kwargs): + self._calls.append({ + 'method': method, + 'args': args, + 'kwargs': kwargs, + }) + call = self.response[method] + if isinstance(call, MultiResponse): + call = call[self.call_count[method]] + self.call_count[method] += 1 + return call + + return callable + else: + return super(ConnMock, self).__getattribute__(method) + + def __call__(self, *args, **kwargs): + return self + + def __enter__(self, *args, **kwargs): + return self + + def __exit__(self, *args, **kwargs): + pass + + def __repr__(self, *args, **kwargs): + return self + + def __getitem__(self, key): + return + + +@contextmanager +def server_mock(response): + with mock.patch('fs.sftpfs.SFTPFS', ConnMock(response)) as SFTPFS: + yield SFTPFS._calls diff --git a/external_file_location/tests/test_sftp.py b/external_file_location/tests/test_sftp.py new file mode 100644 index 00000000000..d08295cb6d4 --- /dev/null +++ b/external_file_location/tests/test_sftp.py @@ -0,0 +1,182 @@ +# coding: utf-8 +# @ 2015 Valentin CHEMIERE @ Akretion +# ©2016 @author Mourad EL HADJ MIMOUNE +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import logging +from StringIO import StringIO +from base64 import b64decode +import hashlib + +import openerp.tests.common as common +from ..tasks.sftp import SftpImportTask +from ..tasks.sftp import SftpExportTask +from .mock_server import (server_mock) +from .mock_server import MultiResponse + + +_logger = logging.getLogger(__name__) + + +class ContextualStringIO(StringIO): + """ + snippet from http://bit.ly/1HfH6uW (stackoverflow) + """ + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + return False + + +class TestNewSource(common.TransactionCase): + def setUp(self): + super(TestNewSource, self).setUp() + self.test_file = ContextualStringIO() + self.test_file.write('import') + self.test_file.seek(0) + self.config = \ + {'file_name': 'testfile', + 'user': 'test', + 'password': 'test', + 'host': 'test', + 'port': 22, + 'attachment_ids': self.env['ir.attachment.metadata'].browse(False) + } + + def test_00_sftp_import(self): + with server_mock( + {'exists': True, + 'makedir': True, + 'open': self.test_file, + 'listdir': ['testfile'] + }): + task = SftpImportTask(self.env, self.config) + task.run() + search_file = self.env['ir.attachment.metadata'].search( + (('name', '=', 'testfile'),)) + self.assertEqual(len(search_file), 1) + self.assertEqual(b64decode(search_file[0].datas), 'import') + + def test_01_sftp_export(self): + with server_mock( + {'isfile': False, + 'open': self.test_file, + }) as FakeSFTP: + task = SftpExportTask(self.env, self.config) + task.run() + if FakeSFTP: + self.assertEqual('open', FakeSFTP[-1]['method']) + + def test_02_sftp_import_delete(self): + with server_mock( + {'exists': True, + 'makedir': True, + 'open': self.test_file, + 'listdir': ['testfile'], + 'remove': True + }) as FakeSFTP: + self.config.update({'after_import': 'delete'}) + task = SftpImportTask(self.env, self.config) + task.run() + search_file = self.env['ir.attachment.metadata'].search( + (('name', '=', 'testfile'),)) + self.assertEqual(len(search_file), 1) + self.assertEqual(b64decode(search_file[0].datas), 'import') + self.assertEqual('remove', FakeSFTP[-1]['method']) + self.assertEqual( + './testfile', FakeSFTP[-1]['args'][0], + "Delete File must be './testfile'") + + def test_03_sftp_import_move(self): + with server_mock( + {'exists': True, + 'makedir': True, + 'open': self.test_file, + 'listdir': ['testfile'], + 'rename': True + }) as FakeSFTP: + self.config.update({'after_import': 'move', 'move_path': '/home'}) + task = SftpImportTask(self.env, self.config) + task.run() + search_file = self.env['ir.attachment.metadata'].search( + (('name', '=', 'testfile'),)) + self.assertEqual(len(search_file), 1) + self.assertEqual(b64decode(search_file[0].datas), 'import') + self.assertEqual('rename', FakeSFTP[-1]['method']) + + def test_04_sftp_import_rename(self): + with server_mock( + {'exists': True, + 'makedir': True, + 'open': self.test_file, + 'listdir': ['testfile'], + 'rename': True + }) as FakeSFTP: + _logger.info("Test sftp rename file") + self.config.update({ + 'after_import': 'rename', + 'new_name': '${obj.name}.imported', + 'path': '/home', + }) + task = SftpImportTask(self.env, self.config) + task.run() + search_file = self.env['ir.attachment.metadata'].search( + (('name', '=', 'testfile'),)) + self.assertEqual(len(search_file), 1) + self.assertEqual(b64decode(search_file[0].datas), 'import') + self.assertEqual('rename', FakeSFTP[2]['method']) + self.assertEqual('/home/testfile.imported', + FakeSFTP[2]['args'][1], + "File not renamed") + + def test_05_sftp_import_move_rename(self): + with server_mock( + {'exists': True, + 'makedir': True, + 'open': self.test_file, + 'listdir': ['testfile'], + 'rename': True + }) as FakeSFTP: + _logger.info("Test sftp move and rename file") + self.config.update({ + 'after_import': 'rename', + 'new_name': '${obj.name}.imported', + 'path': '/home', + 'move_path': '/home/processed', + }) + task = SftpImportTask(self.env, self.config) + task.run() + search_file = self.env['ir.attachment.metadata'].search( + (('name', '=', 'testfile'),)) + self.assertEqual(len(search_file), 1) + self.assertEqual(b64decode(search_file[0].datas), 'import') + self.assertEqual('rename', FakeSFTP[3]['method']) + self.assertEqual('/home/processed/testfile.imported', + FakeSFTP[3]['args'][1], + "File not renamed and moved") + + def test_06_sftp_import_md5(self): + md5_file = ContextualStringIO() + md5_file.write(hashlib.md5('import').hexdigest()) + md5_file.seek(0) + with server_mock( + {'exists': True, + 'makedir': True, + 'open': MultiResponse({ + 1: self.test_file, + 0: md5_file + }), + 'listdir': ['testfile', 'testfile.md5'], + }) as FakeSFTP: + self.config.update({'md5_check': True}) + task = SftpImportTask(self.env, self.config) + task.run() + search_file = self.env['ir.attachment.metadata'].search( + (('name', '=', 'testfile'),)) + self.assertEqual(len(search_file), 1) + self.assertEqual(b64decode(search_file[0].datas), 'import') + self.assertEqual('open', FakeSFTP[-1]['method']) + self.assertEqual('open', FakeSFTP[1]['method']) + self.assertEqual(('./testfile.md5', 'rb'), FakeSFTP[1]['args']) diff --git a/external_file_location/views/attachment_view.xml b/external_file_location/views/attachment_view.xml new file mode 100644 index 00000000000..c2806dbb3db --- /dev/null +++ b/external_file_location/views/attachment_view.xml @@ -0,0 +1,50 @@ + + + + + + ir.attachment.metadata + + + + + + + + + + + + + + ir.attachment.metadata + + + + + + + + + + + + + + ir.attachment.metadata + + + + + + + + + + + + + + + diff --git a/external_file_location/views/location_view.xml b/external_file_location/views/location_view.xml new file mode 100644 index 00000000000..527bcc848f5 --- /dev/null +++ b/external_file_location/views/location_view.xml @@ -0,0 +1,69 @@ + + + + + + external.file.location + +
+ + +
+
+ + + + + + + + + + + + + + + +