<commit_before>import re
import json

import boto3

from flanker.addresslib import address

from inbox.config import config
from inbox.models import Contact
from inbox.models.session import session_scope
from inbox.sqlalchemy_ext.util import safer_yield_per

from sqlalchemy.orm import joinedload

from nylas.logging import get_logger
log = get_logger()

# CloudSearch charges per 1000 batched uploads. Batches must be
# < 5 MB. This assumes that individual items are <= 1kb each.
DOC_UPLOAD_CHUNK_SIZE = 5000

# Be explicit about which fields we search by default.
SEARCH_OPTIONS = '{"fields": ["name", "phone_numbers", "email_address"]}'
search_service_url = config.get('SEARCH_SERVICE_ENDPOINT')
doc_service_url = config.get('DOCUMENT_SERVICE_ENDPOINT')


def get_domain_config(conn, domain_name):
    domains = conn.describe_domains()

    for d in domains['DomainStatusList']:
        if d['DomainName'] == domain_name:
            return d
    return None


def get_search_service():
    return boto3.client(
        "cloudsearchdomain", region_name="us-west-2",
        aws_access_key_id=config.get_required('AWS_ACCESS_KEY_ID'),
        aws_secret_access_key=config.get_required('AWS_SECRET_ACCESS_KEY'),
        endpoint_url='https://{0}'.format(search_service_url))


def get_doc_service():
    return boto3.client(
        "cloudsearchdomain", region_name="us-west-2",
        aws_access_key_id=config.get_required('AWS_ACCESS_KEY_ID'),
        aws_secret_access_key=config.get_required('AWS_SECRET_ACCESS_KEY'),
        endpoint_url='https://{0}'.format(doc_service_url))


def _strip_non_numeric(phone_number):
    digits = [ch for ch in phone_number if re.match('[0-9]', ch)]
    return ''.join(digits)


# CloudSearch doesn't like these characters (reasonably so!)
non_printable_chars_regex = re.compile(
    '[\x01\x02\x03\x04\x05\x06\x07\x08'
    '\x0b\x0c\x0e\x09\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19'
    '\x1a\x1b\x1c\x1d\x1e\x1f]')


def cloudsearch_contact_repr(contact):
    # strip display name out of email address
    parsed = address.parse(contact.email_address)
    name = contact.name or ''
    email_address = parsed.address if parsed else ''
    name_contains_bad_codepoints = re.search(
        non_printable_chars_regex, contact.name or '')
    email_contains_bad_codepoints = re.search(
        non_printable_chars_regex, email_address)
    if name_contains_bad_codepoints or email_contains_bad_codepoints:
        log.warning("bad codepoint in contact", contact_id=contact.id,
                    name=contact.name, email_address=email_address)
        name = non_printable_chars_regex.sub('', name)
        email_address = non_printable_chars_regex.sub('', email_address)
    return {
        'id': contact.id,
        'namespace_id': contact.namespace_id,
        'name': name,
        'email_address': email_address,
        'phone_numbers': [_strip_non_numeric(p.number)
                          for p in contact.phone_numbers]
    }


class ContactSearchClient(object):<commit_msg>Strip out ALL unicode control characters before sending to contact search.<commit_after>import re
import json

import boto3

from flanker.addresslib import address

from inbox.config import config
from inbox.models import Contact
from inbox.models.session import session_scope
from inbox.sqlalchemy_ext.util import safer_yield_per

from sqlalchemy.orm import joinedload

from nylas.logging import get_logger
log = get_logger()

# CloudSearch charges per 1000 batched uploads. Batches must be
# < 5 MB. This assumes that individual items are <= 1kb each.
DOC_UPLOAD_CHUNK_SIZE = 5000

# Be explicit about which fields we search by default.
SEARCH_OPTIONS = '{"fields": ["name", "phone_numbers", "email_address"]}'
search_service_url = config.get('SEARCH_SERVICE_ENDPOINT')
doc_service_url = config.get('DOCUMENT_SERVICE_ENDPOINT')


def get_domain_config(conn, domain_name):
    domains = conn.describe_domains()

    for d in domains['DomainStatusList']:
        if d['DomainName'] == domain_name:
            return d
    return None


def get_search_service():
    return boto3.client(
        "cloudsearchdomain", region_name="us-west-2",
        aws_access_key_id=config.get_required('AWS_ACCESS_KEY_ID'),
        aws_secret_access_key=config.get_required('AWS_SECRET_ACCESS_KEY'),
        endpoint_url='https://{0}'.format(search_service_url))


def get_doc_service():
    return boto3.client(
        "cloudsearchdomain", region_name="us-west-2",
        aws_access_key_id=config.get_required('AWS_ACCESS_KEY_ID'),
        aws_secret_access_key=config.get_required('AWS_SECRET_ACCESS_KEY'),
        endpoint_url='https://{0}'.format(doc_service_url))


def _strip_non_numeric(phone_number):
    digits = [ch for ch in phone_number if re.match('[0-9]', ch)]
    return ''.join(digits)


# Comprehensive list of all unicode control chars (chars for which
# unicodedata.category() returns 'Cc'), which cloudsearch will reject
control_chars_re = re.compile(
    '[\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12'
    '\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f\x80\x81\x82\x83'
    '\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95'
    '\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f]')


def cloudsearch_contact_repr(contact):
    # strip display name out of email address
    parsed = address.parse(contact.email_address)
    name = contact.name or ''
    email_address = parsed.address if parsed else ''
    name_contains_bad_codepoints = re.search(
        control_chars_re, contact.name or '')
    email_contains_bad_codepoints = re.search(
        control_chars_re, email_address)
    if name_contains_bad_codepoints or email_contains_bad_codepoints:
        log.warning("bad codepoint in contact", contact_id=contact.id,
                    name=contact.name, email_address=email_address)
        name = control_chars_re.sub('', name)
        email_address = control_chars_re.sub('', email_address)
    return {
        'id': contact.id,
        'namespace_id': contact.namespace_id,
        'name': name,
        'email_address': email_address,
        'phone_numbers': [_strip_non_numeric(p.number)
                          for p in contact.phone_numbers]
    }


class ContactSearchClient(object):
{'fraction_non_alphanumeric': 0.10602447941503736, 'fraction_numerical': 0.02956604673342871, 'mean_word_length': 3.8850931677018634, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4298858', 'n_tokens_mistral': 2238, 'n_tokens_neox': 2128, 'n_words': 412}