<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: <NAME>
#    Copyright 2013 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import logging
from datetime import datetime
from openerp.tools.translate import _
from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT
from openerp.addons.connector.queue.job import job
from openerp.addons.connector.connector import ConnectorUnit
from openerp.addons.connector.unit.synchronizer import ImportSynchronizer
from openerp.addons.connector.exception import IDMissingInBackend
from ..backend import magento
from ..connector import get_environment, add_checkpoint

_logger = logging.getLogger(__name__)

"""

Importers for Magento.

An import can be skipped if the last sync date is more recent than
the last update in Magento.

They should call the ``bind`` method if the binder even if the records
are already bound, to update the last sync date.

"""

class MagentoImportSynchronizer(ImportSynchronizer):
    """ Base importer for Magento """

    def __init__(self, environment):
        """
        :param environment: current environment (backend, session, ...)
        :type environment: :py:class:`connector.connector.Environment`
        """
        super(MagentoImportSynchronizer, self).__init__(environment)
        self.magento_id = None
        self.magento_record = None

    def _get_magento_data(self):
        """ Return the raw Magento data for ``self.magento_id`` """
        return self.backend_adapter.read(self.magento_id)

    def _before_import(self):
        """ Hook called before the import, when we have the Magento
        data"""

    def _is_uptodate(self, binding_id):
        """Return True if the import should be skipped because
        it is already up-to-date in OpenERP"""
        assert self.magento_record
        if not binding_id:
            return
        binding = self.session.browse(self.model._name, binding_id)
        sync = binding.sync_date
        if not sync:
            return
        fmt = DEFAULT_SERVER_DATETIME_FORMAT
        sync_date = datetime.strptime(sync, fmt)
        magento_date = datetime.strptime(self.magento_record['updated_at'], fmt)
        # if the last synchronization date is greater than the last
        # update in magento, we skip the import.
        # Important: at the beginning of the exporters flows, we have to
        # check if the magento_date is more recent than the sync_date
        # and if so, schedule a new import. If we don't do that, we'll
        # miss changes done in Magento
        return magento_date < sync_date

    def _import_dependencies(self):
        """ Import the dependencies for the record"""
        return

    def _map_data(self):
        """ Call the convert on the Mapper so the converted record can
        be obtained using mapper.data or mapper.data_for_create"""
        self.mapper.convert(self.magento_record)

    def _validate_data(self, data):
        """ Check if the values to import are correct

        Pro-actively check before the ``_create`` or
        ``_update`` if some fields are missing or invalid.

        Raise `InvalidDataError`
        """
        return

    def _get_binding_id(self):
        """Return the binding id from the magento id"""
        return self.binder.to_openerp(self.magento_id)

    def _context(self):
        context = self.session.context.copy()
        context['connector_no_export'] = True
        return context

    def _create(self, data):
        """ Create the OpenERP record """
        with self.session.change_context({'connector_no_export': True}):
            binding_id = self.session.create(self.model._name, data)
        _logger.debug('%s %d created from magento %s',
                      self.model._name, binding_id, self.magento_id)
        return binding_id

    def _update(self, binding_id, data):
        """ Update an OpenERP record """
        with self.session.change_context({'connector_no_export': True}):
            self.session.write(self.model._name, binding_id, data)
        _logger.debug('%s %d updated from magento %s',
                      self.model._name, binding_id, self.magento_id)
        return

    def _after_import(self, binding_id):
        """ Hook called at the end of the import """
        return

    def run(self, magento_id, force=False):
        """ Run the synchronization

        :param magento_id: identifier of the record on Magento
        """
        self.magento_id = magento_id
        try:
            self.magento_record = self._get_magento_data()
        except IDMissingInBackend:
            return _('Record does no longer exist in Magento')
        binding_id = self._get_binding_id()

        if not force and self._is_uptodate(binding_id):
            return _('Already up-to-date.')
        self._before_import()

        # import the missing linked resources
        self._import_dependencies()

        self._map_data()

        if binding_id:
            record = self.mapper.data
            # special check on data before import
            self._validate_data(record)
            self._update(binding_id, record)
        else:
            record = self.mapper.data_for_create
            # special check on data before import
            self._validate_data(record)
            binding_id = self._create(record)

        self.binder.bind(self.magento_id, binding_id)

        self._after_import(binding_id)


class BatchImportSynchronizer(ImportSynchronizer):
    """ The role of a BatchImportSynchronizer is to search for a list of
    items to import, then it can either import them directly or delay
    the import of each item separately.
    """

    def run(self, filters=None):
        """ Run the synchronization """
        record_ids = self.backend_adapter.search(filters)
        for record_id in record_ids:
            self._import_record(record_id)

    def _import_record(self, record_id):
        """ Import a record directly or delay the import of the record.

        Method to implement in sub-classes.
        """
        raise NotImplementedError


class DirectBatchImport(BatchImportSynchronizer):
    """ Import the records directly, without delaying the jobs. """
    _model_name = None

    def _import_record(self, record_id):
        """ Import the record directly """
        import_record(self.session,
                      self.model._name,
                      self.backend_record.id,
                      record_id)


class DelayedBatchImport(BatchImportSynchronizer):
    """ Delay import of the records """
    _model_name = None

    def _import_record(self, record_id, **kwargs):
        """ Delay the import of the records"""
        import_record.delay(self.session,
                            self.model._name,
                            self.backend_record.id,
                            record_id,
                            **kwargs)


@magento
class SimpleRecordImport(MagentoImportSynchronizer):
    """ Import one Magento Website """
    _model_name = [
            'magento.website',
            'magento.res.partner.category',
        ]


@magento
class TranslationImporter(ImportSynchronizer):
    """ Import translations for a record.

    Usually called from importers, in ``_after_import``.
    For instance from the products and products' categories importers.
    """

    _model_name = ['magento.product.category',
                   'magento.product.product',
                   ]

    def _get_magento_data(self, storeview_id=None):
        """ Return the raw Magento data for ``self.magento_id`` """
        return self.backend_adapter.read(self.magento_id, storeview_id)

    def run(self, magento_id, binding_id):
        self.magento_id = magento_id
        session = self.session
        storeview_ids = session.search(
                'magento.storeview',
                [('backend_id', '=', self.backend_record.id)])
        storeviews = session.browse('magento.storeview', storeview_ids)
        default_lang = self.backend_record.default_lang_id
        lang_storeviews = [sv for sv in storeviews
                           if sv.lang_id and sv.lang_id != default_lang]
        if not lang_storeviews:
            return

        # find the translatable fields of the model
        fields = self.model.fields_get(session.cr, session.uid,
                                       context=session.context)
        translatable_fields = [field for field, attrs in fields.iteritems()
                               if attrs.get('translate')]

        for storeview in lang_storeviews:
            lang_record = self._get_magento_data(storeview.magento_id)
            self.mapper.convert(lang_record)
            record = self.mapper.data

            data = dict((field, value) for field, value in record.iteritems()
                        if field in translatable_fields)

            context = session.context.copy()
            context['lang'] = storeview.lang_id.code
            with self.session.change_context({'connector_no_export': True}):
                self.session.write(self.model._name, binding_id, data)


@magento
class AddCheckpoint(ConnectorUnit):
    """ Add a connector.checkpoint on the underlying model
    (not the magento.* but the _inherits'ed model) """

    _model_name = ['magento.product.product',
                   'magento.product.category',
                   'magento.store',
                   ]

    def run(self, openerp_binding_id):
        binding = self.session.browse(self.model._name,
                                      openerp_binding_id)
        record = binding.openerp_id
        add_checkpoint(self.session,
                       record._model._name,
                       record.id,
                       self.backend_record.id)


@job
def import_batch(session, model_name, backend_id, filters=None):
    """ Prepare a batch import of records from Magento """
    env = get_environment(session, model_name, backend_id)
    importer = env.get_connector_unit(BatchImportSynchronizer)
    importer.run(filters=filters)


@job
def import_record(session, model_name, backend_id, magento_id, force=False):
    """ Import a record from Magento """
    env = get_environment(session, model_name, backend_id)
    importer = env.get_connector_unit(MagentoImportSynchronizer)
    importer.run(magento_id, force=force)
<commit_msg>[FIX] Some models in Magento do not have 'log' keys 'updated_at' or 'created_at', we should never assume the key 'updated_at' exist. When it doesn't, the import is never skipped but is always done.<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: <NAME>
#    Copyright 2013 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import logging
from datetime import datetime
from openerp.tools.translate import _
from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT
from openerp.addons.connector.queue.job import job
from openerp.addons.connector.connector import ConnectorUnit
from openerp.addons.connector.unit.synchronizer import ImportSynchronizer
from openerp.addons.connector.exception import IDMissingInBackend
from ..backend import magento
from ..connector import get_environment, add_checkpoint

_logger = logging.getLogger(__name__)

"""

Importers for Magento.

An import can be skipped if the last sync date is more recent than
the last update in Magento.

They should call the ``bind`` method if the binder even if the records
are already bound, to update the last sync date.

"""

class MagentoImportSynchronizer(ImportSynchronizer):
    """ Base importer for Magento """

    def __init__(self, environment):
        """
        :param environment: current environment (backend, session, ...)
        :type environment: :py:class:`connector.connector.Environment`
        """
        super(MagentoImportSynchronizer, self).__init__(environment)
        self.magento_id = None
        self.magento_record = None

    def _get_magento_data(self):
        """ Return the raw Magento data for ``self.magento_id`` """
        return self.backend_adapter.read(self.magento_id)

    def _before_import(self):
        """ Hook called before the import, when we have the Magento
        data"""

    def _is_uptodate(self, binding_id):
        """Return True if the import should be skipped because
        it is already up-to-date in OpenERP"""
        assert self.magento_record
        if not self.magento_record.get('updated_at'):
            return  # no update date on Magento, always import it.
        if not binding_id:
            return  # it does not exist so it shoud not be skipped
        binding = self.session.browse(self.model._name, binding_id)
        sync = binding.sync_date
        if not sync:
            return
        fmt = DEFAULT_SERVER_DATETIME_FORMAT
        sync_date = datetime.strptime(sync, fmt)
        magento_date = datetime.strptime(self.magento_record['updated_at'], fmt)
        # if the last synchronization date is greater than the last
        # update in magento, we skip the import.
        # Important: at the beginning of the exporters flows, we have to
        # check if the magento_date is more recent than the sync_date
        # and if so, schedule a new import. If we don't do that, we'll
        # miss changes done in Magento
        return magento_date < sync_date

    def _import_dependencies(self):
        """ Import the dependencies for the record"""
        return

    def _map_data(self):
        """ Call the convert on the Mapper so the converted record can
        be obtained using mapper.data or mapper.data_for_create"""
        self.mapper.convert(self.magento_record)

    def _validate_data(self, data):
        """ Check if the values to import are correct

        Pro-actively check before the ``_create`` or
        ``_update`` if some fields are missing or invalid.

        Raise `InvalidDataError`
        """
        return

    def _get_binding_id(self):
        """Return the binding id from the magento id"""
        return self.binder.to_openerp(self.magento_id)

    def _context(self):
        context = self.session.context.copy()
        context['connector_no_export'] = True
        return context

    def _create(self, data):
        """ Create the OpenERP record """
        with self.session.change_context({'connector_no_export': True}):
            binding_id = self.session.create(self.model._name, data)
        _logger.debug('%s %d created from magento %s',
                      self.model._name, binding_id, self.magento_id)
        return binding_id

    def _update(self, binding_id, data):
        """ Update an OpenERP record """
        with self.session.change_context({'connector_no_export': True}):
            self.session.write(self.model._name, binding_id, data)
        _logger.debug('%s %d updated from magento %s',
                      self.model._name, binding_id, self.magento_id)
        return

    def _after_import(self, binding_id):
        """ Hook called at the end of the import """
        return

    def run(self, magento_id, force=False):
        """ Run the synchronization

        :param magento_id: identifier of the record on Magento
        """
        self.magento_id = magento_id
        try:
            self.magento_record = self._get_magento_data()
        except IDMissingInBackend:
            return _('Record does no longer exist in Magento')
        binding_id = self._get_binding_id()

        if not force and self._is_uptodate(binding_id):
            return _('Already up-to-date.')
        self._before_import()

        # import the missing linked resources
        self._import_dependencies()

        self._map_data()

        if binding_id:
            record = self.mapper.data
            # special check on data before import
            self._validate_data(record)
            self._update(binding_id, record)
        else:
            record = self.mapper.data_for_create
            # special check on data before import
            self._validate_data(record)
            binding_id = self._create(record)

        self.binder.bind(self.magento_id, binding_id)

        self._after_import(binding_id)


class BatchImportSynchronizer(ImportSynchronizer):
    """ The role of a BatchImportSynchronizer is to search for a list of
    items to import, then it can either import them directly or delay
    the import of each item separately.
    """

    def run(self, filters=None):
        """ Run the synchronization """
        record_ids = self.backend_adapter.search(filters)
        for record_id in record_ids:
            self._import_record(record_id)

    def _import_record(self, record_id):
        """ Import a record directly or delay the import of the record.

        Method to implement in sub-classes.
        """
        raise NotImplementedError


class DirectBatchImport(BatchImportSynchronizer):
    """ Import the records directly, without delaying the jobs. """
    _model_name = None

    def _import_record(self, record_id):
        """ Import the record directly """
        import_record(self.session,
                      self.model._name,
                      self.backend_record.id,
                      record_id)


class DelayedBatchImport(BatchImportSynchronizer):
    """ Delay import of the records """
    _model_name = None

    def _import_record(self, record_id, **kwargs):
        """ Delay the import of the records"""
        import_record.delay(self.session,
                            self.model._name,
                            self.backend_record.id,
                            record_id,
                            **kwargs)


@magento
class SimpleRecordImport(MagentoImportSynchronizer):
    """ Import one Magento Website """
    _model_name = [
            'magento.website',
            'magento.res.partner.category',
        ]


@magento
class TranslationImporter(ImportSynchronizer):
    """ Import translations for a record.

    Usually called from importers, in ``_after_import``.
    For instance from the products and products' categories importers.
    """

    _model_name = ['magento.product.category',
                   'magento.product.product',
                   ]

    def _get_magento_data(self, storeview_id=None):
        """ Return the raw Magento data for ``self.magento_id`` """
        return self.backend_adapter.read(self.magento_id, storeview_id)

    def run(self, magento_id, binding_id):
        self.magento_id = magento_id
        session = self.session
        storeview_ids = session.search(
                'magento.storeview',
                [('backend_id', '=', self.backend_record.id)])
        storeviews = session.browse('magento.storeview', storeview_ids)
        default_lang = self.backend_record.default_lang_id
        lang_storeviews = [sv for sv in storeviews
                           if sv.lang_id and sv.lang_id != default_lang]
        if not lang_storeviews:
            return

        # find the translatable fields of the model
        fields = self.model.fields_get(session.cr, session.uid,
                                       context=session.context)
        translatable_fields = [field for field, attrs in fields.iteritems()
                               if attrs.get('translate')]

        for storeview in lang_storeviews:
            lang_record = self._get_magento_data(storeview.magento_id)
            self.mapper.convert(lang_record)
            record = self.mapper.data

            data = dict((field, value) for field, value in record.iteritems()
                        if field in translatable_fields)

            context = session.context.copy()
            context['lang'] = storeview.lang_id.code
            with self.session.change_context({'connector_no_export': True}):
                self.session.write(self.model._name, binding_id, data)


@magento
class AddCheckpoint(ConnectorUnit):
    """ Add a connector.checkpoint on the underlying model
    (not the magento.* but the _inherits'ed model) """

    _model_name = ['magento.product.product',
                   'magento.product.category',
                   'magento.store',
                   ]

    def run(self, openerp_binding_id):
        binding = self.session.browse(self.model._name,
                                      openerp_binding_id)
        record = binding.openerp_id
        add_checkpoint(self.session,
                       record._model._name,
                       record.id,
                       self.backend_record.id)


@job
def import_batch(session, model_name, backend_id, filters=None):
    """ Prepare a batch import of records from Magento """
    env = get_environment(session, model_name, backend_id)
    importer = env.get_connector_unit(BatchImportSynchronizer)
    importer.run(filters=filters)


@job
def import_record(session, model_name, backend_id, magento_id, force=False):
    """ Import a record from Magento """
    env = get_environment(session, model_name, backend_id)
    importer = env.get_connector_unit(MagentoImportSynchronizer)
    importer.run(magento_id, force=force)

{'fraction_non_alphanumeric': 0.0954944618507568, 'fraction_numerical': 0.0005295441507435682, 'mean_word_length': 2.500463392029657, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2226290', 'n_tokens_mistral': 6196, 'n_tokens_neox': 6050, 'n_words': 1806}