diff --git a/attachment_base_synchronize/security/ir.model.access.csv b/attachment_base_synchronize/security/ir.model.access.csv index a3628dbba29..9d32c4a5359 100644 --- a/attachment_base_synchronize/security/ir.model.access.csv +++ b/attachment_base_synchronize/security/ir.model.access.csv @@ -1,3 +1,3 @@ id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink -access_attachment_metadata_user,ir.attachment.metadata.user,model_ir_attachment_metadata,,1,0,0,0 -access_attachment_metadata_user,ir.attachment.metadata.user,model_ir_attachment_metadata,base.group_no_one,1,1,1,1 +access_attachment_metadata_user_all,ir.attachment.metadata.user,model_ir_attachment_metadata,,1,0,0,0 +access_attachment_metadata_user_conf,ir.attachment.metadata.user,model_ir_attachment_metadata,base.group_no_one,1,1,1,1 diff --git a/external_file_location/README.rst b/external_file_location/README.rst new file mode 100644 index 00000000000..166a2a02039 --- /dev/null +++ b/external_file_location/README.rst @@ -0,0 +1,70 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 + +====================== +External File Location +====================== + +This module was written to extend the functionality of ir.attachment to support remote communication and allow you to import/export file to a remote server. +For now, FTP, SFTP and local filestore are handled by the module. + +Installation +============ + +To install this module, you need to: + +* fs python module at version 0.5.4 or under +* Paramiko python module + +Usage +===== + +To use this module, you need to: + +* Add a location with your server infos +* Create a task with your file info and remote communication method +* A cron task will trigger each task + +.. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas + :alt: Try me on Runbot + :target: https://runbot.odoo-community.org/runbot/149/9.0 + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smashing it by providing a detailed and welcomed feedback. + +Credits +======= + +Images +------ + +* Odoo Community Association: `Icon `_. + +Contributors +------------ + +* Valentin CHEMIERE +* Mourad EL HADJ MIMOUNE +* Florian DA COSTA +* Carlos ALMEIDA + +Maintainer +---------- + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit https://odoo-community.org. diff --git a/external_file_location/__init__.py b/external_file_location/__init__.py new file mode 100644 index 00000000000..cde864bae21 --- /dev/null +++ b/external_file_location/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import models diff --git a/external_file_location/__manifest__.py b/external_file_location/__manifest__.py new file mode 100644 index 00000000000..8fe1065613c --- /dev/null +++ b/external_file_location/__manifest__.py @@ -0,0 +1,35 @@ +# coding: utf-8 +# @ 2016 florian DA COSTA @ Akretion +# © 2016 @author Mourad EL HADJ MIMOUNE +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +{ + 'name': 'External File Location', + 'version': '10.0.1.0.0', + 'author': 'Akretion,Odoo Community Association (OCA)', + 'website': 'http://www.akretion.com/', + 'license': 'AGPL-3', + 'category': 'Generic Modules', + 'depends': [ + 'attachment_base_synchronize', + ], + 'external_dependencies': { + 'python': [ + 'fs', + 'paramiko', + ], + }, + 'data': [ + 'views/menu.xml', + 'views/attachment_view.xml', + 'views/location_view.xml', + 'views/task_view.xml', + 'data/cron.xml', + 'security/ir.model.access.csv', + ], + 'demo': [ + 'demo/task_demo.xml', + ], + 'installable': True, + 'application': False, +} diff --git a/external_file_location/data/cron.xml b/external_file_location/data/cron.xml new file mode 100644 index 00000000000..d24357be93a --- /dev/null +++ b/external_file_location/data/cron.xml @@ -0,0 +1,14 @@ + + + + Run file exchange tasks + 30 + minutes + -1 + True + + external.file.task + run_task_scheduler + ([[('method_type', '=', 'import')]]) + + diff --git a/external_file_location/demo/task_demo.xml b/external_file_location/demo/task_demo.xml new file mode 100644 index 00000000000..f3c02673fc4 --- /dev/null +++ b/external_file_location/demo/task_demo.xml @@ -0,0 +1,95 @@ + + + + TEST FTP + ftp + my-ftp-address + my-ftp-user + my-ftp-password + 21 + + + + TEST SFTP + sftp + my-sftp-address + my-sftp-user + my-sftp-password + 22 + + + + TEST File Store + file_store + / + + + + import + + test-import-ftp.txt + /home/user/test + Import FTP Task + + + + export + + /home/user/test + Export FTP Task + + + + import + + test-import-sftp.txt + /home/user/test + Import SFTP Task + + + + export + + /home/user/test + Export SFTP Task + + + + import + + test-import-filestore.txt + /home/user/test + Import filestore Task + + + + export + + /home/user/test + Export filestore Task + + + + Sftp text export file + dGVzdCBzZnRwIGZpbGUgZXhwb3J0 + sftp_test_export.txt + + export_external_location + + + + ftp text export file + dGVzdCBmdHAgZmlsZSBleHBvcnQ= + ftp_test_export.txt + + export_external_location + + + + filestore text export file + dGVzdCBmaWxlc3RvcmUgZmlsZSBleHBvcnQ= + filestore_test_export.txt + + export_external_location + + diff --git a/external_file_location/models/__init__.py b/external_file_location/models/__init__.py new file mode 100644 index 00000000000..34e38d67f91 --- /dev/null +++ b/external_file_location/models/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +from . import task +from . import attachment +from . import location diff --git a/external_file_location/models/attachment.py b/external_file_location/models/attachment.py new file mode 100644 index 00000000000..1a953caf8fa --- /dev/null +++ b/external_file_location/models/attachment.py @@ -0,0 +1,33 @@ +# coding: utf-8 +# @ 2016 Florian DA COSTA @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import models, fields, api +import base64 +import os + + +class IrAttachmentMetadata(models.Model): + _inherit = 'ir.attachment.metadata' + + task_id = fields.Many2one('external.file.task', string='Task') + location_id = fields.Many2one( + 'external.file.location', string='Location', + related='task_id.location_id', store=True) + file_type = fields.Selection( + selection_add=[ + ('export_external_location', + 'Export File (External location)') + ]) + + @api.multi + def _run(self): + super(IrAttachmentMetadata, self)._run() + if self.file_type == 'export_external_location': + protocols = self.env['external.file.location']._get_classes() + location = self.location_id + cls = protocols.get(location.protocol)[1] + path = os.path.join(self.task_id.filepath, self.datas_fname) + with cls.connect(location) as conn: + datas = base64.decodestring(self.datas) + conn.setcontents(path, data=datas) diff --git a/external_file_location/models/location.py b/external_file_location/models/location.py new file mode 100644 index 00000000000..5af061268be --- /dev/null +++ b/external_file_location/models/location.py @@ -0,0 +1,68 @@ +# coding: utf-8 +# @ 2015 Valentin CHEMIERE @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo import models, fields, api +from ..tasks.filestore import FileStoreTask +from ..tasks.ftp import FtpTask +from ..tasks.sftp import SftpTask + + +class Location(models.Model): + _name = 'external.file.location' + _description = 'Location' + + name = fields.Char(string='Name', required=True) + protocol = fields.Selection(selection='_get_protocol', required=True) + address = fields.Char( + string='Address') + filestore_rootpath = fields.Char( + string='FileStore Root Path', + help="Server's root path") + port = fields.Integer() + login = fields.Char() + password = fields.Char() + task_ids = fields.One2many('external.file.task', 'location_id') + hide_login = fields.Boolean() + hide_password = fields.Boolean() + hide_port = fields.Boolean() + company_id = fields.Many2one( + 'res.company', 'Company', + default=lambda self: self.env['res.company']._company_default_get( + 'external.file.location')) + + @api.model + def _get_classes(self): + "surcharge this method to add new protocols" + return { + 'ftp': ('FTP', FtpTask), + 'sftp': ('SFTP', SftpTask), + 'file_store': ('File Store', FileStoreTask), + } + + @api.model + def _get_protocol(self): + protocols = self._get_classes() + selection = [] + for key, val in protocols.iteritems(): + selection.append((key, val[0])) + return selection + + @api.onchange('protocol') + def onchange_protocol(self): + protocols = self._get_classes() + if self.protocol: + cls = protocols.get(self.protocol)[1] + self.port = cls._default_port + if cls._hide_login: + self.hide_login = True + else: + self.hide_login = False + if cls._hide_password: + self.hide_password = True + else: + self.hide_password = False + if cls._hide_port: + self.hide_port = True + else: + self.hide_port = False diff --git a/external_file_location/models/task.py b/external_file_location/models/task.py new file mode 100644 index 00000000000..eabfe09cba4 --- /dev/null +++ b/external_file_location/models/task.py @@ -0,0 +1,228 @@ +# coding: utf-8 +# @ 2015 Valentin CHEMIERE @ Akretion +# © @author Mourad EL HADJ MIMOUNE +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import datetime +import logging +import os +from base64 import b64encode + +import odoo +from odoo import models, fields, api +from odoo import tools + +_logger = logging.getLogger(__name__) + +try: + # We use a jinja2 sandboxed environment to render mako templates. + # Note that the rendering does not cover all the mako syntax, in particular + # arbitrary Python statements are not accepted, and not all expressions are + # allowed: only "public" attributes (not starting with '_') of objects may + # be accessed. + # This is done on purpose: it prevents incidental or malicious execution of + # Python code that may break the security of the server. + from jinja2.sandbox import SandboxedEnvironment + + mako_template_env = SandboxedEnvironment( + variable_start_string="${", + variable_end_string="}", + line_statement_prefix="%", + trim_blocks=True, # do not output newline after blocks + ) + mako_template_env.globals.update({ + 'str': str, + 'datetime': datetime, + 'len': len, + 'abs': abs, + 'min': min, + 'max': max, + 'sum': sum, + 'filter': filter, + 'reduce': reduce, + 'map': map, + 'round': round, + }) +except ImportError: + _logger.warning("jinja2 not available, templating features will not work!") + + +class Task(models.Model): + _name = 'external.file.task' + _description = 'External file task' + + name = fields.Char(required=True) + + method_type = fields.Selection( + [('import', 'Import'), ('export', 'Export')], + required=True) + + filename = fields.Char(help='File name which is imported.' + 'You can use file pattern like *.txt' + 'to import all txt files') + filepath = fields.Char(help='Path to imported/exported file') + + location_id = fields.Many2one('external.file.location', string='Location', + required=True) + + attachment_ids = fields.One2many('ir.attachment.metadata', 'task_id', + string='Attachment') + + move_path = fields.Char(string='Move Path', + help='Imported File will be moved to this path') + + new_name = fields.Char(string='New Name', + help='Imported File will be renamed to this name' + 'Name can use mako template where obj is an ' + 'ir_attachement. template exemple : ' + ' ${obj.name}-${obj.create_date}.csv') + + md5_check = fields.Boolean(help='Control file integrity after import with' + ' a md5 file') + + after_import = fields.Selection(selection='_get_action', + help='Action after import a file') + + company_id = fields.Many2one( + 'res.company', 'Company', + default=lambda self: self.env['res.company']._company_default_get( + 'external.file.task')) + + file_type = fields.Selection( + selection=[], + string="File Type", + help="The file type determines an import method to be used " + "to parse and transform data before their import in ERP") + + active = fields.Boolean(default=True) + + def _get_action(self): + return [('rename', 'Rename'), + ('move', 'Move'), + ('move_rename', 'Move & Rename'), + ('delete', 'Delete'), + ] + + @api.multi + def _prepare_attachment_vals(self, datas, filename, md5_datas): + self.ensure_one() + vals = { + 'name': filename, + 'datas': b64encode(datas), + 'datas_fname': filename, + 'task_id': self.id, + 'external_hash': md5_datas, + 'file_type': self.file_type or False, + } + return vals + + @api.model + def _template_render(self, template, record): + try: + template = mako_template_env.from_string(tools.ustr(template)) + except Exception: + _logger.exception("Failed to load template %r", template) + + variables = {'obj': record} + try: + render_result = template.render(variables) + except Exception: + _logger.exception( + "Failed to render template %r using values %r" % + (template, variables)) + render_result = u"" + if render_result == u"False": + render_result = u"" + return render_result + + @api.model + def run_task_scheduler(self, domain=None): + if domain is None: + domain = [] + tasks = self.env['external.file.task'].search(domain) + for task in tasks: + if task.method_type == 'import': + task.run_import() + elif task.method_type == 'export': + task.run_export() + + @api.multi + def run_import(self): + self.ensure_one() + protocols = self.env['external.file.location']._get_classes() + cls = protocols.get(self.location_id.protocol)[1] + attach_obj = self.env['ir.attachment.metadata'] + try: + connection = cls.connect(self.location_id) + with connection as conn: + md5_datas = '' + try: + files = conn.listdir(path=self.filepath, + wildcard=self.filename or '', + files_only=True) + for file_name in files: + with api.Environment.manage(): + with odoo.registry( + self.env.cr.dbname).cursor() as new_cr: + new_env = api.Environment(new_cr, self.env.uid, + self.env.context) + try: + full_path = os.path.join(self.filepath, file_name) + file_data = conn.open(full_path, 'rb') + datas = file_data.read() + if self.md5_check: + md5_file = conn.open(full_path + '.md5', 'rb') + md5_datas = md5_file.read().rstrip('\r\n') + attach_vals = self._prepare_attachment_vals( + datas, file_name, md5_datas) + attachment = attach_obj.with_env(new_env).create( + attach_vals) + new_full_path = False + if self.after_import == 'rename': + new_name = self._template_render( + self.new_name, attachment) + new_full_path = os.path.join( + self.filepath, new_name) + elif self.after_import == 'move': + new_full_path = os.path.join( + self.move_path, file_name) + elif self.after_import == 'move_rename': + new_name = self._template_render( + self.new_name, attachment) + new_full_path = os.path.join( + self.move_path, new_name) + if new_full_path: + conn.rename(full_path, new_full_path) + if self.md5_check: + conn.rename( + full_path + '.md5', + new_full_path + '/md5') + if self.after_import == 'delete': + conn.remove(full_path) + if self.md5_check: + conn.remove(full_path + '.md5') + except Exception, e: + new_env.cr.rollback() + _logger.error('Error importing file %s ' + 'from %s: %s', + file_name, + self.filepath, + e) + # move on to process other files + else: + new_env.cr.commit() + except: + _logger.error('Directory %s does not exist', self.filepath) + return + except: + _logger.error('Root directory %s does not exist', self.filepath) + return + + @api.multi + def run_export(self): + self.ensure_one() + attachment_obj = self.env['ir.attachment.metadata'] + attachments = attachment_obj.search( + [('task_id', '=', self.id), ('state', '!=', 'done')]) + for attachment in attachments: + attachment.run() diff --git a/external_file_location/security/ir.model.access.csv b/external_file_location/security/ir.model.access.csv new file mode 100644 index 00000000000..37961f1954f --- /dev/null +++ b/external_file_location/security/ir.model.access.csv @@ -0,0 +1,5 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_external_file_location_manager,external.file.location.manager,model_external_file_location,base.group_system,1,1,1,1 +access_external_file_location_user,external.file.location.user,model_external_file_location,base.group_user,1,0,0,0 +access_external_file_task_manager,external.file.task.manager,model_external_file_task,base.group_system,1,1,1,1 +access_external_file_task_user,external.file.task.user,model_external_file_task,base.group_user,1,0,0,0 diff --git a/external_file_location/tasks/__init__.py b/external_file_location/tasks/__init__.py new file mode 100644 index 00000000000..cf33d19fcfc --- /dev/null +++ b/external_file_location/tasks/__init__.py @@ -0,0 +1,3 @@ +from . import ftp +from . import sftp +from . import filestore diff --git a/external_file_location/tasks/filestore.py b/external_file_location/tasks/filestore.py new file mode 100644 index 00000000000..12c0fd91850 --- /dev/null +++ b/external_file_location/tasks/filestore.py @@ -0,0 +1,27 @@ +# coding: utf-8 +# @ 2016 Florian DA COSTA @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import logging +_logger = logging.getLogger(__name__) + +try: + from fs import osfs +except ImportError: + _logger.debug('Cannot `import fs`.') + + +class FileStoreTask(osfs.OSFS): + + _key = 'filestore' + _name = 'File Store' + _default_port = None + _hide_login = True + _hide_password = True + _hide_port = True + + @staticmethod + def connect(location): + rootpath = location.filestore_rootpath or '/' + conn = FileStoreTask(rootpath) + return conn diff --git a/external_file_location/tasks/ftp.py b/external_file_location/tasks/ftp.py new file mode 100644 index 00000000000..63dacd7d8da --- /dev/null +++ b/external_file_location/tasks/ftp.py @@ -0,0 +1,30 @@ +# coding: utf-8 +# @ 2016 Florian DA COSTA @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import logging +_logger = logging.getLogger(__name__) + +try: + from fs import ftpfs +except ImportError: + _logger.debug('Cannot `import fs`.') + + +class FtpTask(ftpfs.FTPFS): + + _key = 'sftp' + _name = 'SFTP' + _synchronize_type = None + _default_port = 22 + _hide_login = False + _hide_password = False + _hide_port = False + + @staticmethod + def connect(location): + conn = FtpTask(location.address, + location.login, + location.password, + location.port) + return conn diff --git a/external_file_location/tasks/sftp.py b/external_file_location/tasks/sftp.py new file mode 100644 index 00000000000..480193701f7 --- /dev/null +++ b/external_file_location/tasks/sftp.py @@ -0,0 +1,30 @@ +# coding: utf-8 +# @ 2016 Florian DA COSTA @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import logging +_logger = logging.getLogger(__name__) + +try: + from fs import sftpfs +except ImportError: + _logger.debug('Cannot `import fs`.') + + +class SftpTask(sftpfs.SFTPFS): + + _key = 'sftp' + _name = 'SFTP' + _synchronize_type = None + _default_port = 22 + _hide_login = False + _hide_password = False + _hide_port = False + + @staticmethod + def connect(location): + connection_string = "{}:{}".format(location.address, location.port) + conn = SftpTask(connection=connection_string, + username=location.login, + password=location.password) + return conn diff --git a/external_file_location/tests/common.py b/external_file_location/tests/common.py new file mode 100644 index 00000000000..1ba4cf25c5a --- /dev/null +++ b/external_file_location/tests/common.py @@ -0,0 +1,32 @@ +# coding: utf-8 +# @ 2016 Florian da Costa @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import odoo.tests.common as common +from odoo import api +from StringIO import StringIO + + +class ContextualStringIO(StringIO): + """ + snippet from http://bit.ly/1HfH6uW (stackoverflow) + """ + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + return False + + +class TestConnection(common.TransactionCase): + + def setUp(self): + super(TestConnection, self).setUp() + self.registry.enter_test_mode() + self.env = api.Environment(self.registry.test_cr, self.env.uid, + self.env.context) + + def tearDown(self): + self.registry.leave_test_mode() + super(TestConnection, self).tearDown() diff --git a/external_file_location/tests/mock_server.py b/external_file_location/tests/mock_server.py new file mode 100644 index 00000000000..bbf69de2879 --- /dev/null +++ b/external_file_location/tests/mock_server.py @@ -0,0 +1,74 @@ +# coding: utf-8 +# Copyright (C) 2014 initOS GmbH & Co. KG (). +# @ 2015 Valentin CHEMIERE @ Akretion +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +import mock +from contextlib import contextmanager +from collections import defaultdict + + +class MultiResponse(dict): + pass + + +class ConnMock(object): + + def __init__(self, response): + self.response = response + self._calls = [] + self.call_count = defaultdict(int) + + def __getattribute__(self, method): + if method not in ('_calls', 'response', 'call_count'): + def callable(*args, **kwargs): + self._calls.append({ + 'method': method, + 'args': args, + 'kwargs': kwargs, + }) + call = self.response[method] + if isinstance(call, MultiResponse): + call = call[self.call_count[method]] + self.call_count[method] += 1 + return call + + return callable + else: + return super(ConnMock, self).__getattribute__(method) + + def __call__(self, *args, **kwargs): + return self + + def __enter__(self, *args, **kwargs): + return self + + def __exit__(self, *args, **kwargs): + pass + + def __repr__(self, *args, **kwargs): + return self + + def __getitem__(self, key): + return + + +@contextmanager +def server_mock_sftp(response): + with mock.patch('odoo.addons.external_file_location.tasks.sftp.' + 'SftpTask', ConnMock(response)) as SFTPFS: + yield SFTPFS._calls + + +@contextmanager +def server_mock_ftp(response): + with mock.patch('odoo.addons.external_file_location.tasks.ftp.' + 'FtpTask', ConnMock(response)) as FTPFS: + yield FTPFS._calls + + +@contextmanager +def server_mock_filestore(response): + with mock.patch('odoo.addons.external_file_location.tasks.filestore.' + 'FileStoreTask', ConnMock(response)) as FTPFS: + yield FTPFS._calls diff --git a/external_file_location/tests/test_ftp.py b/external_file_location/tests/test_ftp.py new file mode 100644 index 00000000000..b69db89e25c --- /dev/null +++ b/external_file_location/tests/test_ftp.py @@ -0,0 +1,86 @@ +# coding: utf-8 +# @ 2015 Valentin CHEMIERE @ Akretion +# ©2016 @author Mourad EL HADJ MIMOUNE +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import logging +from base64 import b64decode +import hashlib +from .common import TestConnection, ContextualStringIO +from .mock_server import server_mock_ftp +from .mock_server import MultiResponse +from odoo.exceptions import UserError + + +_logger = logging.getLogger(__name__) + + +class TestFtpConnection(TestConnection): + + def setUp(self): + super(TestFtpConnection, self).setUp() + self.test_file_ftp = ContextualStringIO() + self.test_file_ftp.write('import ftp') + self.test_file_ftp.seek(0) + + def test_00_ftp_import(self): + self.task = self.env.ref('external_file_location.ftp_import_task') + with server_mock_ftp( + {'open': self.test_file_ftp, + 'listdir': ['test-import-ftp.txt']}): + self.task.run_import() + search_file = self.env['ir.attachment.metadata'].search( + [('name', '=', 'test-import-ftp.txt')]) + self.assertEqual(len(search_file), 1) + self.assertEqual(b64decode(search_file[0].datas), 'import ftp') + + def test_01_ftp_export(self): + self.task = self.env.ref('external_file_location.ftp_export_task') + self.ftp_attachment = self.env.ref( + 'external_file_location.ir_attachment_export_file_ftp') + with server_mock_ftp( + {'setcontents': ''}) as FakeFTP: + self.task.run_export() + if FakeFTP: + self.assertEqual('setcontents', FakeFTP[-1]['method']) + self.assertEqual('done', self.ftp_attachment.state) + self.assertEqual( + '/home/user/test/ftp_test_export.txt', + FakeFTP[-1]['args'][0]) + self.assertEqual( + 'test ftp file export', + FakeFTP[-1]['kwargs']['data']) + + def test_02_ftp_import_md5(self): + md5_file = ContextualStringIO() + md5_file.write(hashlib.md5('import ftp').hexdigest()) + md5_file.seek(0) + task = self.env.ref('external_file_location.ftp_import_task') + task.md5_check = True + with server_mock_ftp( + {'open': MultiResponse({ + 1: md5_file, + 0: self.test_file_ftp}), + 'listdir': [task.filename]}) as Fakeftp: + task.run_import() + search_file = self.env['ir.attachment.metadata'].search( + (('name', '=', task.filename),)) + self.assertEqual(len(search_file), 1) + self.assertEqual(b64decode(search_file[0].datas), + 'import ftp') + self.assertEqual('open', Fakeftp[-1]['method']) + self.assertEqual(hashlib.md5('import ftp').hexdigest(), + search_file.external_hash) + + def test_03_ftp_import_md5_corrupt_file(self): + md5_file = ContextualStringIO() + md5_file.write(hashlib.md5('import test ftp corrupted').hexdigest()) + md5_file.seek(0) + task = self.env.ref('external_file_location.ftp_import_task') + task.md5_check = True + with server_mock_ftp( + {'open': MultiResponse({ + 1: md5_file, + 0: self.test_file_ftp}), + 'listdir': [task.filename]}): + with self.assertRaises(UserError): + task.run_import() diff --git a/external_file_location/tests/test_sftp.py b/external_file_location/tests/test_sftp.py new file mode 100644 index 00000000000..dea20f661ee --- /dev/null +++ b/external_file_location/tests/test_sftp.py @@ -0,0 +1,85 @@ +# coding: utf-8 +# @ 2015 Valentin CHEMIERE @ Akretion +# ©2016 @author Mourad EL HADJ MIMOUNE +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import logging +from base64 import b64decode +import hashlib +from .common import TestConnection, ContextualStringIO +from .mock_server import server_mock_sftp +from .mock_server import MultiResponse +from odoo.exceptions import UserError + + +_logger = logging.getLogger(__name__) + + +class TestSftpConnection(TestConnection): + + def setUp(self): + super(TestSftpConnection, self).setUp() + self.test_file_sftp = ContextualStringIO() + self.test_file_sftp.write('import sftp') + self.test_file_sftp.seek(0) + + def test_00_sftp_import(self): + task = self.env.ref('external_file_location.sftp_import_task') + with server_mock_sftp( + {'open': self.test_file_sftp, + 'listdir': [task.filename]}): + task.run_import() + search_file = self.env['ir.attachment.metadata'].search( + [('name', '=', task.filename)]) + self.assertEqual(len(search_file), 1) + self.assertEqual(b64decode(search_file[0].datas), 'import sftp') + + def test_01_sftp_export(self): + self.task = self.env.ref('external_file_location.sftp_export_task') + self.sftp_attachment = self.env.ref( + 'external_file_location.ir_attachment_export_file_sftp') + with server_mock_sftp( + {'setcontents': ''}) as FakeSFTP: + self.task.run_export() + if FakeSFTP: + self.assertEqual('setcontents', FakeSFTP[-1]['method']) + self.assertEqual( + '/home/user/test/sftp_test_export.txt', + FakeSFTP[-1]['args'][0]) + self.assertEqual( + 'test sftp file export', + FakeSFTP[-1]['kwargs']['data']) + + def test_02_sftp_import_md5(self): + md5_file = ContextualStringIO() + md5_file.write(hashlib.md5('import sftp').hexdigest()) + md5_file.seek(0) + task = self.env.ref('external_file_location.sftp_import_task') + task.md5_check = True + with server_mock_sftp( + {'open': MultiResponse({ + 1: md5_file, + 0: self.test_file_sftp}), + 'listdir': [task.filename]}) as FakeSFTP: + task.run_import() + search_file = self.env['ir.attachment.metadata'].search( + (('name', '=', task.filename),)) + self.assertEqual(len(search_file), 1) + self.assertEqual(b64decode(search_file[0].datas), + 'import sftp') + self.assertEqual('open', FakeSFTP[-1]['method']) + self.assertEqual(hashlib.md5('import sftp').hexdigest(), + search_file.external_hash) + + def test_03_sftp_import_md5_corrupt_file(self): + md5_file = ContextualStringIO() + md5_file.write(hashlib.md5('import test sftp corrupted').hexdigest()) + md5_file.seek(0) + task = self.env.ref('external_file_location.sftp_import_task') + task.md5_check = True + with server_mock_sftp( + {'open': MultiResponse({ + 1: md5_file, + 0: self.test_file_sftp}), + 'listdir': [task.filename]}): + with self.assertRaises(UserError): + task.run_import() diff --git a/external_file_location/views/attachment_view.xml b/external_file_location/views/attachment_view.xml new file mode 100644 index 00000000000..bbfa9643228 --- /dev/null +++ b/external_file_location/views/attachment_view.xml @@ -0,0 +1,24 @@ + + + + ir.attachment.metadata + + + + + + + + + + + ir.attachment.metadata + + + + + + + + + diff --git a/external_file_location/views/location_view.xml b/external_file_location/views/location_view.xml new file mode 100644 index 00000000000..5b503fb1797 --- /dev/null +++ b/external_file_location/views/location_view.xml @@ -0,0 +1,74 @@ + + + + external.file.location + +
+ + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + external.file.location + + + + + + + + + + + + File Locations + ir.actions.act_window + external.file.location + form + + + + + + diff --git a/external_file_location/views/menu.xml b/external_file_location/views/menu.xml new file mode 100644 index 00000000000..10f3ba4e6d7 --- /dev/null +++ b/external_file_location/views/menu.xml @@ -0,0 +1,8 @@ + + + + diff --git a/external_file_location/views/task_view.xml b/external_file_location/views/task_view.xml new file mode 100644 index 00000000000..d21e28bd1ac --- /dev/null +++ b/external_file_location/views/task_view.xml @@ -0,0 +1,72 @@ + + + + + external.file.task + +
+
+
+ + + +
+
+ + + + + + + + + + + + + + + + + + + +
+ + + + + external.file.task + + + + + + + + + + diff --git a/requirements.txt b/requirements.txt index f86d55f2dcd..a77021bac2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ IPy validate_email pysftp fdb +fs==0.5.4 sqlalchemy raven checksumdir