<commit_before>from __future__ import unicode_literals

import datetime
import json
import logging
import os
import phonenumbers
import regex
import time

from collections import defaultdict
from django.core.exceptions import ValidationError
from django.core.files import File
from django.core.validators import validate_email
from django.db import models, connection
from django.db.models import Count, Max, Q, Sum
from django.utils import timezone
from django.utils.translation import ugettext, ugettext_lazy as _
from guardian.utils import get_anonymous_user
from itertools import chain
from redis_cache import get_redis_connection
from smartmin.models import SmartModel, SmartImportRowError
from smartmin.csv_imports.models import ImportTask
from temba.channels.models import Channel
from temba.orgs.models import Org, OrgLock
from temba.utils.email import send_template_email
from temba.utils import analytics, format_decimal, truncate, datetime_to_str, chunk_list
from temba.utils.models import TembaModel
from temba.utils.exporter import TableExporter
from temba.utils.profiler import SegmentProfiler
from temba.values.models import Value
from temba.locations.models import STATE_LEVEL, DISTRICT_LEVEL, WARD_LEVEL
from uuid import uuid4


logger = logging.getLogger(__name__)

# phone number for every org's test contact
OLD_TEST_CONTACT_TEL = '12065551212'
START_TEST_CONTACT_PATH = 12065550100
END_TEST_CONTACT_PATH = 12065550199

# how many sequential contacts on import triggers suspension
SEQUENTIAL_CONTACTS_THRESHOLD = 250

TEL_SCHEME = 'tel'
TWITTER_SCHEME = 'twitter'
TWILIO_SCHEME = 'twilio'
FACEBOOK_SCHEME = 'facebook'
TELEGRAM_SCHEME = 'telegram'
EMAIL_SCHEME = 'mailto'
EXTERNAL_SCHEME = 'ext'
LINE_SCHEME = 'line'

# Scheme, Label, Export/Import Header, Context Key
URN_SCHEME_CONFIG = ((TEL_SCHEME, _("Phone number"), 'phone', 'tel_e164'),
                     (TWITTER_SCHEME, _("Twitter handle"), 'twitter', TWITTER_SCHEME),
                     (TELEGRAM_SCHEME, _("Telegram identifier"), 'telegram', TELEGRAM_SCHEME),
                     (EMAIL_SCHEME, _("Email address"), 'email', EMAIL_SCHEME),
                     (FACEBOOK_SCHEME, _("Facebook identifier"), 'facebook', FACEBOOK_SCHEME),
                     (EXTERNAL_SCHEME, _("External identifier"), 'external', EXTERNAL_SCHEME),
                     (LINE_SCHEME, _("LINE identifier"), 'line', LINE_SCHEME))

IMPORT_HEADERS = tuple((c[2], c[0]) for c in URN_SCHEME_CONFIG)


class URN(object):
    """
    Support class for URN strings. We differ from the strict definition of a URN (https://tools.ietf.org/html/rfc2141)
    in that:
        * We only supports URNs with scheme and path parts (no netloc, query, params or fragment)
        * Path component can be any non-blank unicode string
        * No hex escaping in URN path
    """
    VALID_SCHEMES = {s[0] for s in URN_SCHEME_CONFIG}

    def __init__(self):  # pragma: no cover
        raise ValueError("Class shouldn't be instantiated")

    @classmethod
    def from_parts(cls, scheme, path):
        """
        Formats a URN scheme and path as single URN string, e.g. tel:+250783835665
        """
        if not scheme or scheme not in cls.VALID_SCHEMES:
            raise ValueError("Invalid scheme component: '%s'" % scheme)

        if not path:
            raise ValueError("Invalid path component: '%s'" % path)

        return '%s:%s' % (scheme, path)

    @classmethod
    def to_parts(cls, urn):
        """
        Parses a URN string (e.g. tel:+250783835665) into a tuple of scheme and path
        """
        try:
            scheme, path = urn.split(':', 1)
        except:
            raise ValueError("URN strings must contain scheme and path components")

        if not scheme or scheme not in cls.VALID_SCHEMES:
            raise ValueError("URN contains an invalid scheme component: '%s'" % scheme)

        if not path:
            raise ValueError("URN contains an invalid path component: '%s'" % path)

        return scheme, path

    @classmethod
    def validate(cls, urn, country_code=None):
        """
        Validates a normalized URN
        """
        try:
            scheme, path = cls.to_parts(urn)
        except ValueError:
            return False

        if scheme == TEL_SCHEME:
            try:
                parsed = phonenumbers.parse(path, country_code)
                return phonenumbers.is_possible_number(parsed)
            except Exception:
                return False

        # validate twitter URNs look like handles
        elif scheme == TWITTER_SCHEME:
            return regex.match(r'^[a-zA-Z0-9_]{1,15}$', path, regex.V0)

        elif scheme == EMAIL_SCHEME:
            try:
                validate_email(path)
                return True
            except ValidationError:
                return False

        # telegram and facebook uses integer ids
        elif scheme in (TELEGRAM_SCHEME, FACEBOOK_SCHEME):
            try:
                int(path)
                return True
            except ValueError:
                return False

        # anything goes for external schemes
        return True

    @classmethod
    def normalize(cls, urn, country_code=None):
        """
        Normalizes the path of a URN string. Should be called anytime looking for a URN match.
        """
        scheme, path = cls.to_parts(urn)

        norm_path = unicode(path).strip()

        if scheme == TEL_SCHEME:
            norm_path, valid = cls.normalize_number(norm_path, country_code)
        elif scheme == TWITTER_SCHEME:
            norm_path = norm_path.lower()
            if norm_path[0:1] == '@':  # strip @ prefix if provided
                norm_path = norm_path[1:]
            norm_path = norm_path.lower()  # Twitter handles are case-insensitive, so we always store as lowercase
        elif scheme == EMAIL_SCHEME or scheme == LINE_SCHEME:
            norm_path = norm_path.lower()

        return cls.from_parts(scheme, norm_path)

    @classmethod
    def normalize_number(cls, number, country_code):
        """
        Normalizes the passed in number, they should be only digits, some backends prepend + and
        maybe crazy users put in dashes or parentheses in the console.

        Returns a tuple of the normalized number and whether it looks like a possible full international
        number.<commit_msg>Ignore lower case urn when line scheme<commit_after>from __future__ import unicode_literals

import datetime
import json
import logging
import os
import phonenumbers
import regex
import time

from collections import defaultdict
from django.core.exceptions import ValidationError
from django.core.files import File
from django.core.validators import validate_email
from django.db import models, connection
from django.db.models import Count, Max, Q, Sum
from django.utils import timezone
from django.utils.translation import ugettext, ugettext_lazy as _
from guardian.utils import get_anonymous_user
from itertools import chain
from redis_cache import get_redis_connection
from smartmin.models import SmartModel, SmartImportRowError
from smartmin.csv_imports.models import ImportTask
from temba.channels.models import Channel
from temba.orgs.models import Org, OrgLock
from temba.utils.email import send_template_email
from temba.utils import analytics, format_decimal, truncate, datetime_to_str, chunk_list
from temba.utils.models import TembaModel
from temba.utils.exporter import TableExporter
from temba.utils.profiler import SegmentProfiler
from temba.values.models import Value
from temba.locations.models import STATE_LEVEL, DISTRICT_LEVEL, WARD_LEVEL
from uuid import uuid4


logger = logging.getLogger(__name__)

# phone number for every org's test contact
OLD_TEST_CONTACT_TEL = '12065551212'
START_TEST_CONTACT_PATH = 12065550100
END_TEST_CONTACT_PATH = 12065550199

# how many sequential contacts on import triggers suspension
SEQUENTIAL_CONTACTS_THRESHOLD = 250

TEL_SCHEME = 'tel'
TWITTER_SCHEME = 'twitter'
TWILIO_SCHEME = 'twilio'
FACEBOOK_SCHEME = 'facebook'
TELEGRAM_SCHEME = 'telegram'
EMAIL_SCHEME = 'mailto'
EXTERNAL_SCHEME = 'ext'
LINE_SCHEME = 'line'

# Scheme, Label, Export/Import Header, Context Key
URN_SCHEME_CONFIG = ((TEL_SCHEME, _("Phone number"), 'phone', 'tel_e164'),
                     (TWITTER_SCHEME, _("Twitter handle"), 'twitter', TWITTER_SCHEME),
                     (TELEGRAM_SCHEME, _("Telegram identifier"), 'telegram', TELEGRAM_SCHEME),
                     (EMAIL_SCHEME, _("Email address"), 'email', EMAIL_SCHEME),
                     (FACEBOOK_SCHEME, _("Facebook identifier"), 'facebook', FACEBOOK_SCHEME),
                     (EXTERNAL_SCHEME, _("External identifier"), 'external', EXTERNAL_SCHEME),
                     (LINE_SCHEME, _("LINE identifier"), 'line', LINE_SCHEME))

IMPORT_HEADERS = tuple((c[2], c[0]) for c in URN_SCHEME_CONFIG)


class URN(object):
    """
    Support class for URN strings. We differ from the strict definition of a URN (https://tools.ietf.org/html/rfc2141)
    in that:
        * We only supports URNs with scheme and path parts (no netloc, query, params or fragment)
        * Path component can be any non-blank unicode string
        * No hex escaping in URN path
    """
    VALID_SCHEMES = {s[0] for s in URN_SCHEME_CONFIG}

    def __init__(self):  # pragma: no cover
        raise ValueError("Class shouldn't be instantiated")

    @classmethod
    def from_parts(cls, scheme, path):
        """
        Formats a URN scheme and path as single URN string, e.g. tel:+250783835665
        """
        if not scheme or scheme not in cls.VALID_SCHEMES:
            raise ValueError("Invalid scheme component: '%s'" % scheme)

        if not path:
            raise ValueError("Invalid path component: '%s'" % path)

        return '%s:%s' % (scheme, path)

    @classmethod
    def to_parts(cls, urn):
        """
        Parses a URN string (e.g. tel:+250783835665) into a tuple of scheme and path
        """
        try:
            scheme, path = urn.split(':', 1)
        except:
            raise ValueError("URN strings must contain scheme and path components")

        if not scheme or scheme not in cls.VALID_SCHEMES:
            raise ValueError("URN contains an invalid scheme component: '%s'" % scheme)

        if not path:
            raise ValueError("URN contains an invalid path component: '%s'" % path)

        return scheme, path

    @classmethod
    def validate(cls, urn, country_code=None):
        """
        Validates a normalized URN
        """
        try:
            scheme, path = cls.to_parts(urn)
        except ValueError:
            return False

        if scheme == TEL_SCHEME:
            try:
                parsed = phonenumbers.parse(path, country_code)
                return phonenumbers.is_possible_number(parsed)
            except Exception:
                return False

        # validate twitter URNs look like handles
        elif scheme == TWITTER_SCHEME:
            return regex.match(r'^[a-zA-Z0-9_]{1,15}$', path, regex.V0)

        elif scheme == EMAIL_SCHEME:
            try:
                validate_email(path)
                return True
            except ValidationError:
                return False

        # telegram and facebook uses integer ids
        elif scheme in (TELEGRAM_SCHEME, FACEBOOK_SCHEME):
            try:
                int(path)
                return True
            except ValueError:
                return False

        # anything goes for external schemes
        return True

    @classmethod
    def normalize(cls, urn, country_code=None):
        """
        Normalizes the path of a URN string. Should be called anytime looking for a URN match.
        """
        scheme, path = cls.to_parts(urn)

        norm_path = unicode(path).strip()

        if scheme == TEL_SCHEME:
            norm_path, valid = cls.normalize_number(norm_path, country_code)
        elif scheme == TWITTER_SCHEME:
            norm_path = norm_path.lower()
            if norm_path[0:1] == '@':  # strip @ prefix if provided
                norm_path = norm_path[1:]
            norm_path = norm_path.lower()  # Twitter handles are case-insensitive, so we always store as lowercase
        elif scheme == EMAIL_SCHEME:
            norm_path = norm_path.lower()

        return cls.from_parts(scheme, norm_path)

    @classmethod
    def normalize_number(cls, number, country_code):
        """
        Normalizes the passed in number, they should be only digits, some backends prepend + and
        maybe crazy users put in dashes or parentheses in the console.

        Returns a tuple of the normalized number and whether it looks like a possible full international
        number.
{'fraction_non_alphanumeric': 0.07792821158690176, 'fraction_numerical': 0.012751889168765743, 'mean_word_length': 2.7312775330396475, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2817290', 'n_tokens_mistral': 3874, 'n_tokens_neox': 3636, 'n_words': 1166}