<commit_before>from __future__ import print_function, unicode_literals

import datetime
import json
import logging
import os
import phonenumbers
import pytz
import regex
import six
import time

from collections import defaultdict
from django.core.exceptions import ValidationError
from django.core.validators import validate_email
from django.db import models
from django.db.models import Count, Max, Q, Sum
from django.utils import timezone
from django.utils.translation import ugettext, ugettext_lazy as _
from guardian.utils import get_anonymous_user
from itertools import chain
from smartmin.models import SmartModel, SmartImportRowError
from smartmin.csv_imports.models import ImportTask
from temba.assets.models import register_asset_store
from temba.channels.models import Channel
from temba.locations.models import AdminBoundary
from temba.orgs.models import Org, OrgLock
from temba.utils import analytics, format_decimal, truncate, datetime_to_str, chunk_list, clean_string
from temba.utils.models import SquashableModel, TembaModel
from temba.utils.export import BaseExportAssetStore, BaseExportTask, TableExporter
from temba.utils.profiler import time_monitor
from temba.values.models import Value


logger = logging.getLogger(__name__)

# phone number for every org's test contact
OLD_TEST_CONTACT_TEL = '12065551212'
START_TEST_CONTACT_PATH = 12065550100
END_TEST_CONTACT_PATH = 12065550199

# how many sequential contacts on import triggers suspension
SEQUENTIAL_CONTACTS_THRESHOLD = 250

EMAIL_SCHEME = 'mailto'
EXTERNAL_SCHEME = 'ext'
FACEBOOK_SCHEME = 'facebook'
LINE_SCHEME = 'line'
TEL_SCHEME = 'tel'
TELEGRAM_SCHEME = 'telegram'
TWILIO_SCHEME = 'twilio'
TWITTER_SCHEME = 'twitter'
VIBER_SCHEME = 'viber'
FCM_SCHEME = 'fcm'

FACEBOOK_PATH_REF_PREFIX = 'ref:'

# Scheme, Label, Export/Import Header, Context Key
URN_SCHEME_CONFIG = ((TEL_SCHEME, _("Phone number"), 'phone', 'tel_e164'),
                     (FACEBOOK_SCHEME, _("Facebook identifier"), 'facebook', FACEBOOK_SCHEME),
                     (TWITTER_SCHEME, _("Twitter handle"), 'twitter', TWITTER_SCHEME),
                     (VIBER_SCHEME, _("Viber identifier"), 'viber', VIBER_SCHEME),
                     (LINE_SCHEME, _("LINE identifier"), 'line', LINE_SCHEME),
                     (TELEGRAM_SCHEME, _("Telegram identifier"), 'telegram', TELEGRAM_SCHEME),
                     (EMAIL_SCHEME, _("Email address"), 'email', EMAIL_SCHEME),
                     (EXTERNAL_SCHEME, _("External identifier"), 'external', EXTERNAL_SCHEME),
                     (FCM_SCHEME, _("Firebase Cloud Messaging identifier"), 'fcm', FCM_SCHEME))

IMPORT_HEADERS = tuple((c[2], c[0]) for c in URN_SCHEME_CONFIG)


class URN(object):
    """
    Support class for URN strings. We differ from the strict definition of a URN (https://tools.ietf.org/html/rfc2141)
    in that:
        * We only supports URNs with scheme and path parts (no netloc, query, params or fragment)
        * Path component can be any non-blank unicode string
        * No hex escaping in URN path
    """
    VALID_SCHEMES = {s[0] for s in URN_SCHEME_CONFIG}

    def __init__(self):  # pragma: no cover
        raise ValueError("Class shouldn't be instantiated")

    @classmethod
    def from_parts(cls, scheme, path):
        """
        Formats a URN scheme and path as single URN string, e.g. tel:+250783835665
        """
        if not scheme or scheme not in cls.VALID_SCHEMES:
            raise ValueError("Invalid scheme component: '%s'" % scheme)

        if not path:
            raise ValueError("Invalid path component: '%s'" % path)

        return '%s:%s' % (scheme, path)

    @classmethod
    def to_parts(cls, urn):
        """
        Parses a URN string (e.g. tel:+250783835665) into a tuple of scheme and path
        """
        try:
            scheme, path = urn.split(':', 1)
        except:
            raise ValueError("URN strings must contain scheme and path components")

        if not scheme or scheme not in cls.VALID_SCHEMES:
            raise ValueError("URN contains an invalid scheme component: '%s'" % scheme)

        if not path:
            raise ValueError("URN contains an invalid path component: '%s'" % path)

        return scheme, path

    @classmethod
    def validate(cls, urn, country_code=None):
        """
        Validates a normalized URN
        """
        try:
            scheme, path = cls.to_parts(urn)
        except ValueError:
            return False

        if scheme == TEL_SCHEME:
            try:
                parsed = phonenumbers.parse(path, country_code)
                return phonenumbers.is_possible_number(parsed)
            except Exception:
                return False

        # validate twitter URNs look like handles
        elif scheme == TWITTER_SCHEME:
            return regex.match(r'^[a-zA-Z0-9_]{1,15}$', path, regex.V0)

        elif scheme == EMAIL_SCHEME:
            try:
                validate_email(path)
                return True
            except ValidationError:
                return False

        # facebook uses integer ids or temp ref ids
        elif scheme == FACEBOOK_SCHEME:
            # we don't validate facebook refs since they come from the outside
            if URN.is_path_fb_ref(path):
                return True

            # otherwise, this should be an int
            else:
                try:
                    int(path)
                    return True
                except ValueError:
                    return False

        # telegram uses integer ids
        elif scheme == TELEGRAM_SCHEME:
            try:
                int(path)
                return True
            except ValueError:
                return False

        # validate Viber URNS look right (this is a guess)
        elif scheme == VIBER_SCHEME:  # pragma: needs cover
            return regex.match(r'^[a-zA-Z0-9_=]{1,16}$', path, regex.V0)

        # anything goes for external schemes
        return True

    @classmethod
    def normalize(cls, urn, country_code=None):
        """
        Normalizes the path of a URN string. Should be called anytime looking for a URN match.
        """
        scheme, path = cls.to_parts(urn)

        norm_path = six.text_type(path).strip()

        if scheme == TEL_SCHEME:
            norm_path, valid = cls.normalize_number(norm_path, country_code)
        elif scheme == TWITTER_SCHEME:
            norm_path = norm_path.lower()
            if norm_path[0:1] == '@':  # strip @ prefix if provided
                norm_path = norm_path[1:]
            norm_path = norm_path.lower()  # Twitter handles are case-insensitive, so we always store as lowercase
        elif scheme == EMAIL_SCHEME:
            norm_path = norm_path.lower()

        return cls.from_parts(scheme, norm_path)

    @classmethod
    def normalize_number(cls, number, country_code):
        """
        Normalizes the passed in number, they should be only digits, some backends prepend + and
        maybe crazy users put in dashes or parentheses in the console.

        Returns a tuple of the normalized number and whether it looks like a possible full international
        number.
        """
        # if the number ends with e11, then that is Excel corrupting it, remove it
        if number.lower().endswith("e+11") or number.lower().endswith("e+12"):
            number = number[0:-4].replace('.', '')

        # remove other characters
        number = regex.sub('[^0-9a-z\+]', '', number.lower(), regex.V0)

        # add on a plus if it looks like it could be a fully qualified number
        if len(number) >= 11 and number[0] not in ['+', '0']:
            number = '+' + number

        normalized = None
        try:
            normalized = phonenumbers.parse(number, str(country_code) if country_code else None)
        except Exception:
            pass

        # now does it look plausible?
        try:
            if phonenumbers.is_possible_number(normalized):
                return phonenumbers.format_number(normalized, phonenumbers.PhoneNumberFormat.E164), True
        except Exception:
            pass

        # this must be a local number of some kind, just lowercase and save
        return regex.sub('[^0-9a-z]', '', number.lower(), regex.V0), False

    @classmethod
    def fb_ref_from_path(cls, path):
        return path[len(FACEBOOK_PATH_REF_PREFIX):]

    @classmethod
    def path_from_fb_ref(cls, ref):
        return FACEBOOK_PATH_REF_PREFIX + ref

    @classmethod
    def is_path_fb_ref(cls, path):
        return path.startswith(FACEBOOK_PATH_REF_PREFIX)

    # ==================== shortcut constructors ===========================

    @classmethod
    def from_tel(cls, path):
        return cls.from_parts(TEL_SCHEME, path)

    @classmethod
    def from_twitter(cls, path):
        return cls.from_parts(TWITTER_SCHEME, path)

    @classmethod
    def from_email(cls, path):
        return cls.from_parts(EMAIL_SCHEME, path)

    @classmethod
    def from_facebook(cls, path):
        return cls.from_parts(FACEBOOK_SCHEME, path)

    @classmethod
    def from_line(cls, path):
        return cls.from_parts(LINE_SCHEME, path)

    @classmethod
    def from_telegram(cls, path):
        return cls.from_parts(TELEGRAM_SCHEME, path)

    @classmethod
    def from_external(cls, path):
        return cls.from_parts(EXTERNAL_SCHEME, path)

    @classmethod
    def from_viber(cls, path):
        return cls.from_parts(VIBER_SCHEME, path)

    @classmethod
    def from_fcm(cls, path):
        return cls.from_parts(FCM_SCHEME, path)


@six.python_2_unicode_compatible
class ContactField(SmartModel):
    """
    Represents a type of field that can be put on Contacts.
    """
    MAX_KEY_LEN = 36
    MAX_LABEL_LEN = 36

    org = models.ForeignKey(Org, verbose_name=_("Org"), related_name="contactfields")

    label = models.CharField(verbose_name=_("Label"), max_length=MAX_LABEL_LEN)

    key = models.CharField(verbose_name=_("Key"), max_length=MAX_KEY_LEN)

    value_type = models.CharField(choices=Value.TYPE_CHOICES, max_length=1, default=Value.TYPE_TEXT,
                                  verbose_name="Field Type")
    show_in_table = models.BooleanField(verbose_name=_("Shown in Tables"), default=False)

    @classmethod
    def make_key(cls, label):
        """
        Generates a key from a label. There is no guarantee that the key is valid so should be checked with is_valid_key
        """
        key = regex.sub(r'([^a-z0-9]+)', ' ', label.lower(), regex.V0)
        return regex.sub(r'([^a-z0-9]+)', '_', key.strip(), regex.V0)

    @classmethod
    def is_valid_key(cls, key):
        return regex.match(r'^[a-z][a-z0-9_]*$', key, regex.V0) and key not in Contact.RESERVED_FIELDS and len(key) <= cls.MAX_KEY_LEN

    @classmethod
    def is_valid_label(cls, label):
        label = label.strip()
        return regex.match(r'^[A-Za-z0-9\- ]+$', label, regex.V0) and len(label) <= cls.MAX_LABEL_LEN

    @classmethod
    def hide_field(cls, org, user, key):
        existing = ContactField.objects.filter(org=org, key=key).first()
        if existing:
            existing.is_active = False
            existing.show_in_table = False
            existing.modified_by = user
            existing.save(update_fields=('is_active', 'show_in_table', 'modified_by', 'modified_on'))

            # cancel any events on this
            from temba.campaigns.models import EventFire
            EventFire.update_field_events(existing)

    @classmethod
    def get_or_create(cls, org, user, key, label=None, show_in_table=None, value_type=None):
        """
        Gets the existing contact field or creates a new field if it doesn't exist
        """
        if label:
            label = label.strip()

        with org.lock_on(OrgLock.field, key):
            field = ContactField.objects.filter(org=org, key__iexact=key).first()

            if not field:
                # try to lookup the existing field by label
                field = ContactField.get_by_label(org, label)

            # we have a field with a invalid key we should ignore it
            if field and not ContactField.is_valid_key(field.key):
                field = None

            if field:
                update_events = False
                changed = False

                # make this as active
                if not field.is_active:
                    field.is_active = True
                    update_events = True
                    changed = True

                # update whether we show in tables if passed in
                if show_in_table is not None and show_in_table != field.show_in_table:
                    field.show_in_table = show_in_table
                    changed = True

                # update our label if we were given one
                if label and field.label != label:
                    field.label = label
                    changed = True

                # update our type if we were given one
                if value_type and field.value_type != value_type:
                    field.value_type = value_type
                    changed = True

                if changed:
                    field.modified_by = user
                    field.save()

                    if update_events:
                        from temba.campaigns.models import EventFire
                        EventFire.update_field_events(field)

            else:
                # we need to create a new contact field, use our key with invalid chars removed
                if not label:
                    label = regex.sub(r'([^A-Za-z0-9\- ]+)', ' ', key, regex.V0).title()

                if not value_type:
                    value_type = Value.TYPE_TEXT

                if show_in_table is None:
                    show_in_table = False

                if not ContactField.is_valid_key(key):
                    raise ValueError('Field key %s has invalid characters or is a reserved field name' % key)

                field = ContactField.objects.create(org=org, key=key, label=label,
                                                    show_in_table=show_in_table, value_type=value_type,
                                                    created_by=user, modified_by=user)

            return field

    @classmethod
    def get_by_label(cls, org, label):
        return cls.objects.filter(org=org, is_active=True, label__iexact=label).first()

    @classmethod
    def get_location_field(cls, org, type):
        return cls.objects.filter(is_active=True, org=org, value_type=type).first()

    def __str__(self):
        return "%s" % self.label


NEW_CONTACT_VARIABLE = "@new_contact"


@six.python_2_unicode_compatible
class Contact(TembaModel):
    name = models.CharField(verbose_name=_("Name"), max_length=128, blank=True, null=True,
                            help_text=_("The name of this contact"))

    org = models.ForeignKey(Org, verbose_name=_("Org"), related_name="org_contacts",
                            help_text=_("The organization that this contact belongs to"))

    is_blocked = models.BooleanField(verbose_name=_("Is Blocked"), default=False,
                                     help_text=_("Whether this contact has been blocked"))

    is_test = models.BooleanField(verbose_name=_("Is Test"), default=False,
                                  help_text=_("Whether this contact is for simulation"))

    is_stopped = models.BooleanField(verbose_name=_("Is Stopped"), default=False,
                                     help_text=_("Whether this contact has opted out of receiving messages"))

    language = models.CharField(max_length=3, verbose_name=_("Language"), null=True, blank=True,
                                help_text=_("The preferred language for this contact"))

    simulation = False

    NAME = 'name'
    FIRST_NAME = 'first_name'
    LANGUAGE = 'language'
    PHONE = 'phone'
    UUID = 'uuid'
    GROUPS = 'groups'
    ID = 'id'

    # reserved contact fields
    RESERVED_FIELDS = [
        NAME, FIRST_NAME, PHONE, LANGUAGE, GROUPS, UUID, 'created_by', 'modified_by', 'org', 'is', 'has'
    ] + [c[0] for c in IMPORT_HEADERS]

    @classmethod
    def get_contacts(cls, org, blocked=False, stopped=False):
        return Contact.objects.filter(org=org, is_active=True, is_test=False, is_blocked=blocked, is_stopped=stopped)

    @property<commit_msg>Remove no longer used Contact.get_contactsy<commit_after>from __future__ import print_function, unicode_literals

import datetime
import json
import logging
import os
import phonenumbers
import pytz
import regex
import six
import time

from collections import defaultdict
from django.core.exceptions import ValidationError
from django.core.validators import validate_email
from django.db import models
from django.db.models import Count, Max, Q, Sum
from django.utils import timezone
from django.utils.translation import ugettext, ugettext_lazy as _
from guardian.utils import get_anonymous_user
from itertools import chain
from smartmin.models import SmartModel, SmartImportRowError
from smartmin.csv_imports.models import ImportTask
from temba.assets.models import register_asset_store
from temba.channels.models import Channel
from temba.locations.models import AdminBoundary
from temba.orgs.models import Org, OrgLock
from temba.utils import analytics, format_decimal, truncate, datetime_to_str, chunk_list, clean_string
from temba.utils.models import SquashableModel, TembaModel
from temba.utils.export import BaseExportAssetStore, BaseExportTask, TableExporter
from temba.utils.profiler import time_monitor
from temba.values.models import Value


logger = logging.getLogger(__name__)

# phone number for every org's test contact
OLD_TEST_CONTACT_TEL = '12065551212'
START_TEST_CONTACT_PATH = 12065550100
END_TEST_CONTACT_PATH = 12065550199

# how many sequential contacts on import triggers suspension
SEQUENTIAL_CONTACTS_THRESHOLD = 250

EMAIL_SCHEME = 'mailto'
EXTERNAL_SCHEME = 'ext'
FACEBOOK_SCHEME = 'facebook'
LINE_SCHEME = 'line'
TEL_SCHEME = 'tel'
TELEGRAM_SCHEME = 'telegram'
TWILIO_SCHEME = 'twilio'
TWITTER_SCHEME = 'twitter'
VIBER_SCHEME = 'viber'
FCM_SCHEME = 'fcm'

FACEBOOK_PATH_REF_PREFIX = 'ref:'

# Scheme, Label, Export/Import Header, Context Key
URN_SCHEME_CONFIG = ((TEL_SCHEME, _("Phone number"), 'phone', 'tel_e164'),
                     (FACEBOOK_SCHEME, _("Facebook identifier"), 'facebook', FACEBOOK_SCHEME),
                     (TWITTER_SCHEME, _("Twitter handle"), 'twitter', TWITTER_SCHEME),
                     (VIBER_SCHEME, _("Viber identifier"), 'viber', VIBER_SCHEME),
                     (LINE_SCHEME, _("LINE identifier"), 'line', LINE_SCHEME),
                     (TELEGRAM_SCHEME, _("Telegram identifier"), 'telegram', TELEGRAM_SCHEME),
                     (EMAIL_SCHEME, _("Email address"), 'email', EMAIL_SCHEME),
                     (EXTERNAL_SCHEME, _("External identifier"), 'external', EXTERNAL_SCHEME),
                     (FCM_SCHEME, _("Firebase Cloud Messaging identifier"), 'fcm', FCM_SCHEME))

IMPORT_HEADERS = tuple((c[2], c[0]) for c in URN_SCHEME_CONFIG)


class URN(object):
    """
    Support class for URN strings. We differ from the strict definition of a URN (https://tools.ietf.org/html/rfc2141)
    in that:
        * We only supports URNs with scheme and path parts (no netloc, query, params or fragment)
        * Path component can be any non-blank unicode string
        * No hex escaping in URN path
    """
    VALID_SCHEMES = {s[0] for s in URN_SCHEME_CONFIG}

    def __init__(self):  # pragma: no cover
        raise ValueError("Class shouldn't be instantiated")

    @classmethod
    def from_parts(cls, scheme, path):
        """
        Formats a URN scheme and path as single URN string, e.g. tel:+250783835665
        """
        if not scheme or scheme not in cls.VALID_SCHEMES:
            raise ValueError("Invalid scheme component: '%s'" % scheme)

        if not path:
            raise ValueError("Invalid path component: '%s'" % path)

        return '%s:%s' % (scheme, path)

    @classmethod
    def to_parts(cls, urn):
        """
        Parses a URN string (e.g. tel:+250783835665) into a tuple of scheme and path
        """
        try:
            scheme, path = urn.split(':', 1)
        except:
            raise ValueError("URN strings must contain scheme and path components")

        if not scheme or scheme not in cls.VALID_SCHEMES:
            raise ValueError("URN contains an invalid scheme component: '%s'" % scheme)

        if not path:
            raise ValueError("URN contains an invalid path component: '%s'" % path)

        return scheme, path

    @classmethod
    def validate(cls, urn, country_code=None):
        """
        Validates a normalized URN
        """
        try:
            scheme, path = cls.to_parts(urn)
        except ValueError:
            return False

        if scheme == TEL_SCHEME:
            try:
                parsed = phonenumbers.parse(path, country_code)
                return phonenumbers.is_possible_number(parsed)
            except Exception:
                return False

        # validate twitter URNs look like handles
        elif scheme == TWITTER_SCHEME:
            return regex.match(r'^[a-zA-Z0-9_]{1,15}$', path, regex.V0)

        elif scheme == EMAIL_SCHEME:
            try:
                validate_email(path)
                return True
            except ValidationError:
                return False

        # facebook uses integer ids or temp ref ids
        elif scheme == FACEBOOK_SCHEME:
            # we don't validate facebook refs since they come from the outside
            if URN.is_path_fb_ref(path):
                return True

            # otherwise, this should be an int
            else:
                try:
                    int(path)
                    return True
                except ValueError:
                    return False

        # telegram uses integer ids
        elif scheme == TELEGRAM_SCHEME:
            try:
                int(path)
                return True
            except ValueError:
                return False

        # validate Viber URNS look right (this is a guess)
        elif scheme == VIBER_SCHEME:  # pragma: needs cover
            return regex.match(r'^[a-zA-Z0-9_=]{1,16}$', path, regex.V0)

        # anything goes for external schemes
        return True

    @classmethod
    def normalize(cls, urn, country_code=None):
        """
        Normalizes the path of a URN string. Should be called anytime looking for a URN match.
        """
        scheme, path = cls.to_parts(urn)

        norm_path = six.text_type(path).strip()

        if scheme == TEL_SCHEME:
            norm_path, valid = cls.normalize_number(norm_path, country_code)
        elif scheme == TWITTER_SCHEME:
            norm_path = norm_path.lower()
            if norm_path[0:1] == '@':  # strip @ prefix if provided
                norm_path = norm_path[1:]
            norm_path = norm_path.lower()  # Twitter handles are case-insensitive, so we always store as lowercase
        elif scheme == EMAIL_SCHEME:
            norm_path = norm_path.lower()

        return cls.from_parts(scheme, norm_path)

    @classmethod
    def normalize_number(cls, number, country_code):
        """
        Normalizes the passed in number, they should be only digits, some backends prepend + and
        maybe crazy users put in dashes or parentheses in the console.

        Returns a tuple of the normalized number and whether it looks like a possible full international
        number.
        """
        # if the number ends with e11, then that is Excel corrupting it, remove it
        if number.lower().endswith("e+11") or number.lower().endswith("e+12"):
            number = number[0:-4].replace('.', '')

        # remove other characters
        number = regex.sub('[^0-9a-z\+]', '', number.lower(), regex.V0)

        # add on a plus if it looks like it could be a fully qualified number
        if len(number) >= 11 and number[0] not in ['+', '0']:
            number = '+' + number

        normalized = None
        try:
            normalized = phonenumbers.parse(number, str(country_code) if country_code else None)
        except Exception:
            pass

        # now does it look plausible?
        try:
            if phonenumbers.is_possible_number(normalized):
                return phonenumbers.format_number(normalized, phonenumbers.PhoneNumberFormat.E164), True
        except Exception:
            pass

        # this must be a local number of some kind, just lowercase and save
        return regex.sub('[^0-9a-z]', '', number.lower(), regex.V0), False

    @classmethod
    def fb_ref_from_path(cls, path):
        return path[len(FACEBOOK_PATH_REF_PREFIX):]

    @classmethod
    def path_from_fb_ref(cls, ref):
        return FACEBOOK_PATH_REF_PREFIX + ref

    @classmethod
    def is_path_fb_ref(cls, path):
        return path.startswith(FACEBOOK_PATH_REF_PREFIX)

    # ==================== shortcut constructors ===========================

    @classmethod
    def from_tel(cls, path):
        return cls.from_parts(TEL_SCHEME, path)

    @classmethod
    def from_twitter(cls, path):
        return cls.from_parts(TWITTER_SCHEME, path)

    @classmethod
    def from_email(cls, path):
        return cls.from_parts(EMAIL_SCHEME, path)

    @classmethod
    def from_facebook(cls, path):
        return cls.from_parts(FACEBOOK_SCHEME, path)

    @classmethod
    def from_line(cls, path):
        return cls.from_parts(LINE_SCHEME, path)

    @classmethod
    def from_telegram(cls, path):
        return cls.from_parts(TELEGRAM_SCHEME, path)

    @classmethod
    def from_external(cls, path):
        return cls.from_parts(EXTERNAL_SCHEME, path)

    @classmethod
    def from_viber(cls, path):
        return cls.from_parts(VIBER_SCHEME, path)

    @classmethod
    def from_fcm(cls, path):
        return cls.from_parts(FCM_SCHEME, path)


@six.python_2_unicode_compatible
class ContactField(SmartModel):
    """
    Represents a type of field that can be put on Contacts.
    """
    MAX_KEY_LEN = 36
    MAX_LABEL_LEN = 36

    org = models.ForeignKey(Org, verbose_name=_("Org"), related_name="contactfields")

    label = models.CharField(verbose_name=_("Label"), max_length=MAX_LABEL_LEN)

    key = models.CharField(verbose_name=_("Key"), max_length=MAX_KEY_LEN)

    value_type = models.CharField(choices=Value.TYPE_CHOICES, max_length=1, default=Value.TYPE_TEXT,
                                  verbose_name="Field Type")
    show_in_table = models.BooleanField(verbose_name=_("Shown in Tables"), default=False)

    @classmethod
    def make_key(cls, label):
        """
        Generates a key from a label. There is no guarantee that the key is valid so should be checked with is_valid_key
        """
        key = regex.sub(r'([^a-z0-9]+)', ' ', label.lower(), regex.V0)
        return regex.sub(r'([^a-z0-9]+)', '_', key.strip(), regex.V0)

    @classmethod
    def is_valid_key(cls, key):
        return regex.match(r'^[a-z][a-z0-9_]*$', key, regex.V0) and key not in Contact.RESERVED_FIELDS and len(key) <= cls.MAX_KEY_LEN

    @classmethod
    def is_valid_label(cls, label):
        label = label.strip()
        return regex.match(r'^[A-Za-z0-9\- ]+$', label, regex.V0) and len(label) <= cls.MAX_LABEL_LEN

    @classmethod
    def hide_field(cls, org, user, key):
        existing = ContactField.objects.filter(org=org, key=key).first()
        if existing:
            existing.is_active = False
            existing.show_in_table = False
            existing.modified_by = user
            existing.save(update_fields=('is_active', 'show_in_table', 'modified_by', 'modified_on'))

            # cancel any events on this
            from temba.campaigns.models import EventFire
            EventFire.update_field_events(existing)

    @classmethod
    def get_or_create(cls, org, user, key, label=None, show_in_table=None, value_type=None):
        """
        Gets the existing contact field or creates a new field if it doesn't exist
        """
        if label:
            label = label.strip()

        with org.lock_on(OrgLock.field, key):
            field = ContactField.objects.filter(org=org, key__iexact=key).first()

            if not field:
                # try to lookup the existing field by label
                field = ContactField.get_by_label(org, label)

            # we have a field with a invalid key we should ignore it
            if field and not ContactField.is_valid_key(field.key):
                field = None

            if field:
                update_events = False
                changed = False

                # make this as active
                if not field.is_active:
                    field.is_active = True
                    update_events = True
                    changed = True

                # update whether we show in tables if passed in
                if show_in_table is not None and show_in_table != field.show_in_table:
                    field.show_in_table = show_in_table
                    changed = True

                # update our label if we were given one
                if label and field.label != label:
                    field.label = label
                    changed = True

                # update our type if we were given one
                if value_type and field.value_type != value_type:
                    field.value_type = value_type
                    changed = True

                if changed:
                    field.modified_by = user
                    field.save()

                    if update_events:
                        from temba.campaigns.models import EventFire
                        EventFire.update_field_events(field)

            else:
                # we need to create a new contact field, use our key with invalid chars removed
                if not label:
                    label = regex.sub(r'([^A-Za-z0-9\- ]+)', ' ', key, regex.V0).title()

                if not value_type:
                    value_type = Value.TYPE_TEXT

                if show_in_table is None:
                    show_in_table = False

                if not ContactField.is_valid_key(key):
                    raise ValueError('Field key %s has invalid characters or is a reserved field name' % key)

                field = ContactField.objects.create(org=org, key=key, label=label,
                                                    show_in_table=show_in_table, value_type=value_type,
                                                    created_by=user, modified_by=user)

            return field

    @classmethod
    def get_by_label(cls, org, label):
        return cls.objects.filter(org=org, is_active=True, label__iexact=label).first()

    @classmethod
    def get_location_field(cls, org, type):
        return cls.objects.filter(is_active=True, org=org, value_type=type).first()

    def __str__(self):
        return "%s" % self.label


NEW_CONTACT_VARIABLE = "@new_contact"


@six.python_2_unicode_compatible
class Contact(TembaModel):
    name = models.CharField(verbose_name=_("Name"), max_length=128, blank=True, null=True,
                            help_text=_("The name of this contact"))

    org = models.ForeignKey(Org, verbose_name=_("Org"), related_name="org_contacts",
                            help_text=_("The organization that this contact belongs to"))

    is_blocked = models.BooleanField(verbose_name=_("Is Blocked"), default=False,
                                     help_text=_("Whether this contact has been blocked"))

    is_test = models.BooleanField(verbose_name=_("Is Test"), default=False,
                                  help_text=_("Whether this contact is for simulation"))

    is_stopped = models.BooleanField(verbose_name=_("Is Stopped"), default=False,
                                     help_text=_("Whether this contact has opted out of receiving messages"))

    language = models.CharField(max_length=3, verbose_name=_("Language"), null=True, blank=True,
                                help_text=_("The preferred language for this contact"))

    simulation = False

    NAME = 'name'
    FIRST_NAME = 'first_name'
    LANGUAGE = 'language'
    PHONE = 'phone'
    UUID = 'uuid'
    GROUPS = 'groups'
    ID = 'id'

    # reserved contact fields
    RESERVED_FIELDS = [
        NAME, FIRST_NAME, PHONE, LANGUAGE, GROUPS, UUID, 'created_by', 'modified_by', 'org', 'is', 'has'
    ] + [c[0] for c in IMPORT_HEADERS]

    @property
{'fraction_non_alphanumeric': 0.09163799459326616, 'fraction_numerical': 0.008232981076431556, 'mean_word_length': 2.304537610394884, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3592901', 'n_tokens_mistral': 9843, 'n_tokens_neox': 9304, 'n_words': 2623}