<commit_before>from collections import defaultdict
import re
import json
from dateutil.parser import parse
from datetime import datetime, timedelta
from decimal import Decimal
from couchdbkit.ext.django.schema import *
from couchdbkit.exceptions import MultipleResultsFound
from dimagi.utils.couch.undo import DELETED_SUFFIX
from dimagi.utils.decorators.memoized import memoized
from django.conf import settings
from dimagi.utils.couch.database import get_safe_write_kwargs
from dimagi.utils.modules import try_import
from dimagi.utils.parsing import json_format_datetime
from corehq.apps.domain.models import Domain
from couchdbkit import ResourceNotFound

phone_number_re = re.compile("^\d+$")

class PhoneNumberException(Exception):
    pass

class InvalidFormatException(PhoneNumberException):
    pass

class PhoneNumberInUseException(PhoneNumberException):
    pass

class BadSMSConfigException(Exception):
    pass


class BackendProcessingException(Exception):
    pass


class UnrecognizedBackendException(Exception):
    pass


class VerifiedNumber(Document):
    """
    There should only be one VerifiedNumber entry per (owner_doc_type, owner_id), and
    each VerifiedNumber.phone_number should be unique across all entries.
    """
    domain          = StringProperty()
    owner_doc_type  = StringProperty()
    owner_id        = StringProperty()
    phone_number    = StringProperty()
    backend_id      = StringProperty() # the name of a MobileBackend (can be domain-level or system-level)
    ivr_backend_id  = StringProperty() # points to a MobileBackend
    verified        = BooleanProperty()
    
    def __repr__(self):
        return '{phone} in {domain} (owned by {owner})'.format(
            phone=self.phone_number, domain=self.domain,
            owner=self.owner_id
        )

    @property
    def backend(self):
        from corehq.apps.sms.util import clean_phone_number
        if self.backend_id is not None and isinstance(self.backend_id, basestring) and self.backend_id.strip() != "":
            return MobileBackend.load_by_name(self.domain, self.backend_id)
        else:
            return MobileBackend.auto_load(clean_phone_number(self.phone_number), self.domain)

    @property
    def ivr_backend(self):
        return MobileBackend.get(self.ivr_backend_id)
    
    @property
    def owner(self):
        if self.owner_doc_type == "CommCareCase":
            # Circular import
            from corehq.apps.sms.models import CommConnectCase
            return CommConnectCase.get(self.owner_id)
        elif self.owner_doc_type == "CommCareUser":
            # Circular import
            from corehq.apps.users.models import CommCareUser
            return CommCareUser.get(self.owner_id)
        elif self.owner_doc_type == 'WebUser':
            # Circular importsms
            from corehq.apps.users.models import WebUser
            return WebUser.get(self.owner_id)
        else:
            return None

    def retire(self, deletion_id=None):
        self.doc_type += DELETED_SUFFIX
        if deletion_id:
            self['-deletion_id'] = deletion_id

        self.save()

    @classmethod
    def by_extensive_search(cls, phone_number):
        # Try to look up the verified number entry directly
        v = cls.by_phone(phone_number)

        # If not found, try to see if any number in the database is a substring
        # of the number given to us. This can happen if the telco prepends some
        # international digits, such as 011...
        if v is None:
            v = cls.by_phone(phone_number[1:])
        if v is None:
            v = cls.by_phone(phone_number[2:])
        if v is None:
            v = cls.by_phone(phone_number[3:])

        # If still not found, try to match only the last digits of numbers in 
        # the database. This can happen if the telco removes the country code
        # in the caller id.
        if v is None:
            v = cls.by_suffix(phone_number)

        return v

    @classmethod
    def by_phone(cls, phone_number, include_pending=False):
        return cls.phone_lookup(
            "sms/verified_number_by_number",
            phone_number,
            include_pending
        )

    @classmethod
    def by_suffix(cls, phone_number, include_pending=False):
        """
        Used to lookup a VerifiedNumber, trying to exclude country code digits.
        """
        try:
            result = cls.phone_lookup(
                "sms/verified_number_by_suffix",
                phone_number,
                include_pending
            )
        except MultipleResultsFound:
            # We can't pinpoint who the number belongs to because more than one
            # suffix matches. So treat it as if the result was not found.
            result = None
        return result

    @classmethod
    def phone_lookup(cls, view_name, phone_number, include_pending=False):
        # We use .one() here because the framework prevents duplicates
        # from being entered when a contact saves a number.
        # See CommCareMobileContactMixin.save_verified_number()
        from corehq.apps.sms.util import strip_plus
        v = cls.view(view_name,
                     key=strip_plus(phone_number),
                     include_docs=True).one()
        return v if (include_pending or (v and v.verified)) else None

    @classmethod
    def by_domain(cls, domain):
        result = cls.view("sms/verified_number_by_domain",
                          startkey=[domain],
                          endkey=[domain, {}],
                          include_docs=True,
                          reduce=False).all()
        return result

def add_plus(phone_number):
    return ('+' + phone_number) if not phone_number.startswith('+') else phone_number

def get_global_prefix_backend_mapping():
    result = {}
    for entry in BackendMapping.view("sms/backend_map", startkey=["*"], endkey=["*", {}], include_docs=True).all():
        if entry.prefix == "*":
            result[""] = entry.backend_id
        else:
            result[entry.prefix] = entry.backend_id
    return result

class MobileBackend(Document):
    """
    Defines an instance of a backend api to be used for either sending sms, or sending outbound calls.
    """
    class Meta:
        app_label = "sms"

    base_doc = "MobileBackend"
    domain = StringProperty()               # This is the domain that the backend belongs to, or None for global backends
    name = StringProperty()                 # The name to use when setting this backend for a contact
    display_name = StringProperty()        # Simple name to display to users - e.g. Twilio
    authorized_domains = ListProperty(StringProperty)  # A list of additional domains that are allowed to use this backend
    is_global = BooleanProperty(default=True)  # If True, this backend can be used for any domain
    description = StringProperty()          # (optional) A description of this backend<commit_msg>add incoming_api_id attr to mobilebackend<commit_after>from collections import defaultdict
import re
import json
from dateutil.parser import parse
from datetime import datetime, timedelta
from decimal import Decimal
from couchdbkit.ext.django.schema import *
from couchdbkit.exceptions import MultipleResultsFound
from dimagi.utils.couch.undo import DELETED_SUFFIX
from dimagi.utils.decorators.memoized import memoized
from django.conf import settings
from dimagi.utils.couch.database import get_safe_write_kwargs
from dimagi.utils.modules import try_import
from dimagi.utils.parsing import json_format_datetime
from corehq.apps.domain.models import Domain
from couchdbkit import ResourceNotFound

phone_number_re = re.compile("^\d+$")

class PhoneNumberException(Exception):
    pass

class InvalidFormatException(PhoneNumberException):
    pass

class PhoneNumberInUseException(PhoneNumberException):
    pass

class BadSMSConfigException(Exception):
    pass


class BackendProcessingException(Exception):
    pass


class UnrecognizedBackendException(Exception):
    pass


class VerifiedNumber(Document):
    """
    There should only be one VerifiedNumber entry per (owner_doc_type, owner_id), and
    each VerifiedNumber.phone_number should be unique across all entries.
    """
    domain          = StringProperty()
    owner_doc_type  = StringProperty()
    owner_id        = StringProperty()
    phone_number    = StringProperty()
    backend_id      = StringProperty() # the name of a MobileBackend (can be domain-level or system-level)
    ivr_backend_id  = StringProperty() # points to a MobileBackend
    verified        = BooleanProperty()
    
    def __repr__(self):
        return '{phone} in {domain} (owned by {owner})'.format(
            phone=self.phone_number, domain=self.domain,
            owner=self.owner_id
        )

    @property
    def backend(self):
        from corehq.apps.sms.util import clean_phone_number
        if self.backend_id is not None and isinstance(self.backend_id, basestring) and self.backend_id.strip() != "":
            return MobileBackend.load_by_name(self.domain, self.backend_id)
        else:
            return MobileBackend.auto_load(clean_phone_number(self.phone_number), self.domain)

    @property
    def ivr_backend(self):
        return MobileBackend.get(self.ivr_backend_id)
    
    @property
    def owner(self):
        if self.owner_doc_type == "CommCareCase":
            # Circular import
            from corehq.apps.sms.models import CommConnectCase
            return CommConnectCase.get(self.owner_id)
        elif self.owner_doc_type == "CommCareUser":
            # Circular import
            from corehq.apps.users.models import CommCareUser
            return CommCareUser.get(self.owner_id)
        elif self.owner_doc_type == 'WebUser':
            # Circular importsms
            from corehq.apps.users.models import WebUser
            return WebUser.get(self.owner_id)
        else:
            return None

    def retire(self, deletion_id=None):
        self.doc_type += DELETED_SUFFIX
        if deletion_id:
            self['-deletion_id'] = deletion_id

        self.save()

    @classmethod
    def by_extensive_search(cls, phone_number):
        # Try to look up the verified number entry directly
        v = cls.by_phone(phone_number)

        # If not found, try to see if any number in the database is a substring
        # of the number given to us. This can happen if the telco prepends some
        # international digits, such as 011...
        if v is None:
            v = cls.by_phone(phone_number[1:])
        if v is None:
            v = cls.by_phone(phone_number[2:])
        if v is None:
            v = cls.by_phone(phone_number[3:])

        # If still not found, try to match only the last digits of numbers in 
        # the database. This can happen if the telco removes the country code
        # in the caller id.
        if v is None:
            v = cls.by_suffix(phone_number)

        return v

    @classmethod
    def by_phone(cls, phone_number, include_pending=False):
        return cls.phone_lookup(
            "sms/verified_number_by_number",
            phone_number,
            include_pending
        )

    @classmethod
    def by_suffix(cls, phone_number, include_pending=False):
        """
        Used to lookup a VerifiedNumber, trying to exclude country code digits.
        """
        try:
            result = cls.phone_lookup(
                "sms/verified_number_by_suffix",
                phone_number,
                include_pending
            )
        except MultipleResultsFound:
            # We can't pinpoint who the number belongs to because more than one
            # suffix matches. So treat it as if the result was not found.
            result = None
        return result

    @classmethod
    def phone_lookup(cls, view_name, phone_number, include_pending=False):
        # We use .one() here because the framework prevents duplicates
        # from being entered when a contact saves a number.
        # See CommCareMobileContactMixin.save_verified_number()
        from corehq.apps.sms.util import strip_plus
        v = cls.view(view_name,
                     key=strip_plus(phone_number),
                     include_docs=True).one()
        return v if (include_pending or (v and v.verified)) else None

    @classmethod
    def by_domain(cls, domain):
        result = cls.view("sms/verified_number_by_domain",
                          startkey=[domain],
                          endkey=[domain, {}],
                          include_docs=True,
                          reduce=False).all()
        return result

def add_plus(phone_number):
    return ('+' + phone_number) if not phone_number.startswith('+') else phone_number

def get_global_prefix_backend_mapping():
    result = {}
    for entry in BackendMapping.view("sms/backend_map", startkey=["*"], endkey=["*", {}], include_docs=True).all():
        if entry.prefix == "*":
            result[""] = entry.backend_id
        else:
            result[entry.prefix] = entry.backend_id
    return result

class MobileBackend(Document):
    """
    Defines an instance of a backend api to be used for either sending sms, or sending outbound calls.
    """
    class Meta:
        app_label = "sms"

    base_doc = "MobileBackend"
    domain = StringProperty()               # This is the domain that the backend belongs to, or None for global backends
    name = StringProperty()                 # The name to use when setting this backend for a contact
    display_name = StringProperty()         # Simple name to display to users - e.g. Twilio
    incoming_api_id = StringProperty()      # Some Gateways have different API ids for IN/OUT
    authorized_domains = ListProperty(StringProperty)  # A list of additional domains that are allowed to use this backend
    is_global = BooleanProperty(default=True)  # If True, this backend can be used for any domain
    description = StringProperty()          # (optional) A description of this backend
{'fraction_non_alphanumeric': 0.07428245851435084, 'fraction_numerical': 0.0008546399829072003, 'mean_word_length': 2.52991452991453, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2452809', 'n_tokens_mistral': 3864, 'n_tokens_neox': 3798, 'n_words': 1151}