<commit_before>"""Provide iCloud contacts"""

from inbox.log import get_logger
logger = get_logger()

from carddav import CardDav
import lxml.etree as ET
from inbox.contacts.vcard import vcard_from_string
from inbox.contacts.carddav import supports_carddav

from inbox.models.session import session_scope
from inbox.models import Contact
from inbox.models.backends.generic import GenericAccount


ICLOUD_CONTACTS_URL = 'https://contacts.icloud.com/'


class ICloudContactsProvider(object):
    """
    Base class to fetch and parse iCloud contacts
    """

    PROVIDER_NAME = 'icloud'

    def __init__(self, account_id, namespace_id):
        supports_carddav(ICLOUD_CONTACTS_URL)
        self.account_id = account_id
        self.namespace_id = namespace_id
        self.log = logger.new(account_id=account_id, component='contacts sync',
                              provider=self.PROVIDER_NAME)

    def _vCard_raw_to_contact(self, cardstring):
        card = vcard_from_string(cardstring)

        def _x(key):  # Ugly parsing helper for ugly formats
            if key in card:
                try:
                    return card[key][0][0]
                except IndexError:
                    pass

        # Skip contact groups for now
        if _x('X-ADDRESSBOOKSERVER-KIND') == 'group':
            return None

        uid = _x('UID')
        name = _x('FN')
        email_address = _x('EMAIL')
        # TODO add these later
        # street_address = _x('ADR')
        # phone_number = _x('TEL')
        # organization = _x('ORG')

        return Contact(namespace_id=self.namespace_id,
                       provider_name=self.PROVIDER_NAME,
                       source='remote',
                       uid=uid,
                       name=name,
                       email_address=email_address,
                       raw_data=cardstring)

    def get_items(self, sync_from_dt=None, max_results=100000):
        with session_scope() as db_session:
            account = db_session.query(GenericAccount).get(self.account_id)
            email_address = account.email_address
            password = account.password
            if account.provider != 'icloud':
                self.log.error("Can't sync contacts for non iCloud provider",
                               account_id=account.id,
                               provider=account.provider)
                return []

        c = CardDav(email_address, password, ICLOUD_CONTACTS_URL)

        # Get the `principal` URL for the users's CardDav endpont
        principal = c.get_principal_url()

        # Get addressbook home URL on user's specific iCloud shard/subdomain
        home_url = c.get_address_book_home(ICLOUD_CONTACTS_URL + principal)
        self.log.info("Home URL for user's contacts: {}".format(home_url))
        self.log.debug("Requesting cards for user")

        # This request is limited to returning 5000 items
        returned_cards = c.get_cards(home_url + 'card/')

        root = ET.XML(returned_cards)

        all_contacts = []
        for refprop in root.iterchildren():

            try:
                cardstring = refprop[1][0][1].text
            except IndexError:
                # This can happen when there are errors or other responses.
                # Currently if there are over 5000 contacts, it trigger the
                # response number-of-matches-within-limits
                # TODO add paging for requesting all
                self.log.error("Error parsing CardDav response into contact: "
                               "{}".format(ET.tostring(refprop)))
                continue

            new_contact = self._vCard_raw_to_contact(cardstring)
            if new_contact:
                all_contacts.append(new_contact)

        self.log.info("Saving {} contacts from iCloud sync"
                      .format(len(all_contacts)))
        return all_contacts
<commit_msg>Fix typo in iCloud contacts sync.<commit_after>"""Provide iCloud contacts"""

from inbox.log import get_logger
logger = get_logger()

from carddav import CardDav
import lxml.etree as ET
from inbox.contacts.vcard import vcard_from_string
from inbox.contacts.carddav import supports_carddav

from inbox.models.session import session_scope
from inbox.models import Contact
from inbox.models.backends.generic import GenericAccount


ICLOUD_CONTACTS_URL = 'https://contacts.icloud.com/'


class ICloudContactsProvider(object):
    """
    Base class to fetch and parse iCloud contacts
    """

    PROVIDER_NAME = 'icloud'

    def __init__(self, account_id, namespace_id):
        supports_carddav(ICLOUD_CONTACTS_URL)
        self.account_id = account_id
        self.namespace_id = namespace_id
        self.log = logger.new(account_id=account_id, component='contacts sync',
                              provider=self.PROVIDER_NAME)

    def _vCard_raw_to_contact(self, cardstring):
        card = vcard_from_string(cardstring)

        def _x(key):  # Ugly parsing helper for ugly formats
            if key in card:
                try:
                    return card[key][0][0]
                except IndexError:
                    pass

        # Skip contact groups for now
        if _x('X-ADDRESSBOOKSERVER-KIND') == 'group':
            return None

        uid = _x('UID')
        name = _x('FN')
        email_address = _x('EMAIL')
        # TODO add these later
        # street_address = _x('ADR')
        # phone_number = _x('TEL')
        # organization = _x('ORG')

        return Contact(namespace_id=self.namespace_id,
                       provider_name=self.PROVIDER_NAME,
                       uid=uid,
                       name=name,
                       email_address=email_address,
                       raw_data=cardstring)

    def get_items(self, sync_from_dt=None, max_results=100000):
        with session_scope() as db_session:
            account = db_session.query(GenericAccount).get(self.account_id)
            email_address = account.email_address
            password = <PASSWORD>
            if account.provider != 'icloud':
                self.log.error("Can't sync contacts for non iCloud provider",
                               account_id=account.id,
                               provider=account.provider)
                return []

        c = CardDav(email_address, password, ICLOUD_CONTACTS_URL)

        # Get the `principal` URL for the users's CardDav endpont
        principal = c.get_principal_url()

        # Get addressbook home URL on user's specific iCloud shard/subdomain
        home_url = c.get_address_book_home(ICLOUD_CONTACTS_URL + principal)
        self.log.info("Home URL for user's contacts: {}".format(home_url))
        self.log.debug("Requesting cards for user")

        # This request is limited to returning 5000 items
        returned_cards = c.get_cards(home_url + 'card/')

        root = ET.XML(returned_cards)

        all_contacts = []
        for refprop in root.iterchildren():

            try:
                cardstring = refprop[1][0][1].text
            except IndexError:
                # This can happen when there are errors or other responses.
                # Currently if there are over 5000 contacts, it trigger the
                # response number-of-matches-within-limits
                # TODO add paging for requesting all
                self.log.error("Error parsing CardDav response into contact: "
                               "{}".format(ET.tostring(refprop)))
                continue

            new_contact = self._vCard_raw_to_contact(cardstring)
            if new_contact:
                all_contacts.append(new_contact)

        self.log.info("Saving {} contacts from iCloud sync"
                      .format(len(all_contacts)))
        return all_contacts

{'fraction_non_alphanumeric': 0.08055413032324268, 'fraction_numerical': 0.0048742945100051305, 'mean_word_length': 2.0104247104247106, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6925686', 'n_tokens_mistral': 2232, 'n_tokens_neox': 2100, 'n_words': 520}