<commit_before># -*- coding: UTF-8 -*-
# Copyright 2014-2015 <NAME>
# License: BSD (see file COPYING for details)

"""Defines the :class:`Dupable` model mixin and related functionality
to assist users in finding duplicate database records.

The current implementation uses a helper table with "phonetic words"
and the `Double Metaphone
<https://en.wikipedia.org/wiki/Metaphone#Double_Metaphone>`_ of the
detection phonetic algorithm (from the `fuzzy
<https://pypi.python.org/pypi/Fuzzy>`_ module).  Read also Doug
Hellmann about `Using Fuzzy Matching to Search by Sound with Python
<http://www.informit.com/articles/article.aspx?p=1848528>`_
(2012-03-22).

Note about the name: to dupe *somebody* means "to make a dupe of;
deceive; delude; trick." (`reference.com
<http://dictionary.reference.com/browse/dupe>`_), while to dupe
*something* means to duplicate it (eventually in order to cheat
somebody e.g. by making a cheap copy of a valuable object).

Don't mix up this module with :mod:`lino.mixins.duplicable`.  Models
are "duplicable" if users --sometimes-- *want* to duplicate some
instance thereof.

"""

from __future__ import unicode_literals
from builtins import map
from builtins import str
from builtins import object

from django.conf import settings
from django.db import models

from lino.api import dd, _
from lino.core.actions import SubmitInsert
from lino.utils import join_elems
from lino.utils.xmlgen.html import E


class CheckedSubmitInsert(SubmitInsert):
    """Like the standard :class:`lino.core.actions.SubmitInsert`, but adds
    a confirmation if there is a possible duplicate record.

    """
    def run_from_ui(self, ar, **kw):
        obj = ar.create_instance_from_request()

        def ok(ar2):
            self.save_new_instance(ar2, obj)
            ar2.set_response(close_window=True)
            # logger.info("20140512 CheckedSubmitInsert")

        qs = list(obj.find_similar_instances(4))
        if len(qs) > 0:
            msg = _("There are %d similar %s:") % (
                len(qs), obj._meta.verbose_name_plural)
            for other in qs:
                msg += '<br/>\n' + str(other)

            msg += '<br/>\n'
            msg += _("Are you sure you want to create a new "
                     "%(model)s named %(name)s?") % dict(
                model=obj._meta.verbose_name,
                name=obj.get_full_name())

            ar.confirm(ok, msg)
        else:
            ok(ar)


from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class PhoneticWordBase(dd.Model):
    """Base class for the table of phonetic words of a given dupable
    model. For every (non-abstract) dupable model there must be a
    subclass of `PhoneticWordBase`.
    The subclass must define a field
    :attr:`owner` which points to the `Dupable`, and the `Dupable`'s
    :attr:`dupable_word_model` must point to its subclass
    of `PhoneticWordBase`.

    """
    class Meta(object):
        abstract = True

    allow_cascaded_delete = ['owner']

    word = models.CharField(max_length=100)

    def __str__(self):
        return self.word

    @classmethod
    def on_analyze(cls, site):
        import fuzzy
        cls._fuzzy_DMetaphone = fuzzy.DMetaphone()

    @classmethod
    def reduce_word(cls, s):
        # fuzzy.DMetaphone does not work with unicode strings, see
        # https://bitbucket.org/yougov/fuzzy/issue/2/fuzzy-support-for-unicode-strings-with
        dm = cls._fuzzy_DMetaphone(s.encode('utf8'))
        dms = dm[0] or dm[1]
        if dms is None:
            return ''
        return dms.decode('utf8')


class Dupable(dd.Model):
    """Base class for models that can be "dupable".

    This mixin is to be used on models for which there is a danger of
    having unwanted duplicate records. It is both for *avoiding* such
    duplicates on new records and for *detecting* existing duplicates.

    Note that adding :class:`Dupable` to your model's base classes does
    not yet activate any functionality, it just declares that model as
    being dupable.  In order to activate verification, you must also
    define a model which implements :class:`PhoneticWordBase` and set
    :attr:`Dupable.dupable_word_model` to point to that model.  This is
    done by plugins like :mod:`lino_xl.lib.dupable_partners` or
    :mod:`lino_welfare.modlib.dupable_clients`

    """
    class Meta(object):<commit_msg>replace fuzzy by metaphone which supports Py3.<commit_after># -*- coding: UTF-8 -*-
# Copyright 2014-2015 <NAME>
# License: BSD (see file COPYING for details)

"""Defines the :class:`Dupable` model mixin and related functionality
to assist users in finding duplicate database records.

The current implementation uses a helper table with "phonetic words"
and the `Double Metaphone
<https://en.wikipedia.org/wiki/Metaphone#Double_Metaphone>`_ of the
detection phonetic algorithm (from the `fuzzy
<https://pypi.python.org/pypi/Fuzzy>`_ module).  Read also <NAME> about `Using Fuzzy Matching to Search by Sound with Python
<http://www.informit.com/articles/article.aspx?p=1848528>`_
(2012-03-22).

Note about the name: to dupe *somebody* means "to make a dupe of;
deceive; delude; trick." (`reference.com
<http://dictionary.reference.com/browse/dupe>`_), while to dupe
*something* means to duplicate it (eventually in order to cheat
somebody e.g. by making a cheap copy of a valuable object).

Don't mix up this module with :mod:`lino.mixins.duplicable`.  Models
are "duplicable" if users --sometimes-- *want* to duplicate some
instance thereof.

"""

from __future__ import unicode_literals
from builtins import map
from builtins import str
from builtins import object

from django.conf import settings
from django.db import models

from lino.api import dd, _
from lino.core.actions import SubmitInsert
from lino.utils import join_elems
from lino.utils.xmlgen.html import E


class CheckedSubmitInsert(SubmitInsert):
    """Like the standard :class:`lino.core.actions.SubmitInsert`, but adds
    a confirmation if there is a possible duplicate record.

    """
    def run_from_ui(self, ar, **kw):
        obj = ar.create_instance_from_request()

        def ok(ar2):
            self.save_new_instance(ar2, obj)
            ar2.set_response(close_window=True)
            # logger.info("20140512 CheckedSubmitInsert")

        qs = list(obj.find_similar_instances(4))
        if len(qs) > 0:
            msg = _("There are %d similar %s:") % (
                len(qs), obj._meta.verbose_name_plural)
            for other in qs:
                msg += '<br/>\n' + str(other)

            msg += '<br/>\n'
            msg += _("Are you sure you want to create a new "
                     "%(model)s named %(name)s?") % dict(
                model=obj._meta.verbose_name,
                name=obj.get_full_name())

            ar.confirm(ok, msg)
        else:
            ok(ar)


from django.utils.encoding import python_2_unicode_compatible


@python_2_unicode_compatible
class PhoneticWordBase(dd.Model):
    """Base class for the table of phonetic words of a given dupable
    model. For every (non-abstract) dupable model there must be a
    subclass of `PhoneticWordBase`.
    The subclass must define a field
    :attr:`owner` which points to the `Dupable`, and the `Dupable`'s
    :attr:`dupable_word_model` must point to its subclass
    of `PhoneticWordBase`.

    """
    class Meta(object):
        abstract = True

    allow_cascaded_delete = ['owner']

    word = models.CharField(max_length=100)

    def __str__(self):
        return self.word

    @classmethod
    def on_analyze(cls, site):
        import metaphone as fuzzy
        cls._fuzzy_DMetaphone = fuzzy.doublemetaphone
        # import fuzzy
        # cls._fuzzy_DMetaphone = fuzzy.DMetaphone()

    @classmethod
    def reduce_word(cls, s):
        # fuzzy.DMetaphone does not work with unicode strings, see
        # https://bitbucket.org/yougov/fuzzy/issue/2/fuzzy-support-for-unicode-strings-with
        dm = cls._fuzzy_DMetaphone(s.encode('utf8'))
        dms = dm[0] or dm[1]
        if dms is None:
            return ''
        return dms.decode('utf8')


class Dupable(dd.Model):
    """Base class for models that can be "dupable".

    This mixin is to be used on models for which there is a danger of
    having unwanted duplicate records. It is both for *avoiding* such
    duplicates on new records and for *detecting* existing duplicates.

    Note that adding :class:`Dupable` to your model's base classes does
    not yet activate any functionality, it just declares that model as
    being dupable.  In order to activate verification, you must also
    define a model which implements :class:`PhoneticWordBase` and set
    :attr:`Dupable.dupable_word_model` to point to that model.  This is
    done by plugins like :mod:`lino_xl.lib.dupable_partners` or
    :mod:`lino_welfare.modlib.dupable_clients`

    """
    class Meta(object):
{'fraction_non_alphanumeric': 0.09552572706935122, 'fraction_numerical': 0.010626398210290829, 'mean_word_length': 3.0714936247723132, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 20, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2094139', 'n_tokens_mistral': 2879, 'n_tokens_neox': 2669, 'n_words': 898}