<commit_before>import logging
import time
from datetime import datetime
from datetime import timedelta
from urlparse import urljoin

import pytz
import requests

from exceptions import ResponsysClientError
from utils import convert_to_list_of_dicts
from utils import convert_to_table_structure

logger = logging.getLogger(__name__)


class ResponsysClient(object):

    AUTH_TOKEN_REFRESH_THRESHOLD = timedelta(hours=1)
    AUTH_PATH = '/rest/api/v1.1/auth/token'
    RESPONSYS_RATE_LIMIT_WAITING_PERIOD_IN_SECONDS = 60
    RESPONSYS_INVALID_TOKEN_RESPONSE_DETAIL = 'Not a valid authentication token'
    DEFAULT_REQUEST_TIMEOUT_IN_SECONDS = 60
    RECORD_PROCESS_LIMIT_QUANTITY = 200

    def __init__(self, username=None,
                 password=<PASSWORD>,
                 login_url=None,
                 profile_list=None,
                 folder_name=None,
                 rental_supp_table=None):

        self.username = username
        self.password = password
        self.login_url = login_url
        self.issued_url = None
        self.auth_token = None
        self.refresh_timestamp = None
        self.profile_list = profile_list
        self.folder = folder_name
        self.rental_supp_table = rental_supp_table

    def get_profile_contact_lists(self):
        method = 'GET'
        path = '/rest/api/v1.1/lists'

        response = self.send_request(method, path)

        self._check_for_valid_response(method, path, response.status_code, response.text)

        return response.json()

    def get_extension_tables_lists(self):
        method = 'GET'
        path = '/rest/api/v1.1/lists/{}/listExtensions'.format(self.profile_list)

        response = self.send_request(method, path)

        self._check_for_valid_response(method, path, response.status_code, response.text)

        return response.json()

    def create_or_update_contact_list_members(self, contact_dicts):
        member_field_names, member_records = convert_to_table_structure(contact_dicts)

        path = '/rest/api/v1.1/lists/{}/members'.format(self.profile_list)

        response = self.send_merge_request(path, member_field_names, member_records)

        self._check_for_valid_response(response.request.method, path, response.status_code,
                                       response.text)

        return response.json()

    def create_or_update_extension_table_members(self, table_name, data_dicts):
        member_field_names, member_records = convert_to_table_structure(data_dicts)

        path = ('/rest/api/v1.1/lists/{}/listExtensions/{}/members'
                .format(self.profile_list, table_name))

        response = self.send_ext_table_merge_request(path, member_field_names, member_records)

        self._check_for_valid_response(response.request.method,
                                       path, response.status_code, response.text)

        return response.json()

    def create_or_update_supplemental_table_members(self, table_name, data_dicts):
        member_field_names, member_records = convert_to_table_structure(data_dicts)

        path = ('/rest/api/v1.1/folders/{}/suppData/{}/members'
                .format(self.folder, table_name))

        response = self.send_supp_table_merge_request(path, member_field_names, member_records)

        self._check_for_valid_response(response.request.method, path, response.status_code,
                                       response.text)

        return response.json()

    def get_contact_list_table_member(self, user_id):
        method = 'GET'
        path = '/rest/api/v1.1/lists/{}/members/'.format(self.profile_list)
        query_by_customer_id_params = {
            'qa': 'c',
            'id': str(user_id),
            'fs': 'all'
        }

        response = self.send_request(method, path, params=query_by_customer_id_params)

        self._check_for_valid_response(method, path, response.status_code, response.text)

        members = self._parse_members(response)

        return members[0]

    def get_extension_table_member(self, table_name, user_id):
        method = 'GET'
        path = ('/rest/api/v1.1/lists/{}/listExtensions/{}/members'
                .format(self.profile_list, table_name))
        query_by_customer_id_params = {
            'qa': 'c',
            'id': str(user_id),
            'fs': 'all'
        }

        response = self.send_request(method, path, params=query_by_customer_id_params)

        self._check_for_valid_response(method, path, response.status_code, response.text)

        members = self._parse_members(response)

        return members[0]

    def delete_contact_list_member(self, user_id):
        member = self.get_contact_list_table_member(user_id)

        method = 'DELETE'
        path = '/rest/api/v1.1/lists/{}/members/{}'.format(self.profile_list, member['RIID_'])

        response = self.send_request(method, path)

        self._check_for_valid_response(method, path, response.status_code, response.text)

    def delete_extension_table_member(self, table_name, user_id):
        member = self.get_extension_table_member(table_name, user_id)

        method = 'DELETE'
        path = ('/rest/api/v1.1/lists/{}/listExtensions/{}/members/{}'
                .format(self.profile_list, table_name, member['RIID_']))

        response = self.send_request(method, path)

        self._check_for_valid_response(method, path, response.status_code, response.text)

    @staticmethod
    def _parse_members(response):
        parsed_response = response.json()

        field_names = parsed_response['recordData']['fieldNames']
        records = parsed_response['recordData']['records']

        members = convert_to_list_of_dicts(field_names, records)

        return members

    def send_request(self, method, path, params=None, json=None):
        self._get_access_items()

        url = urljoin(self.issued_url, path)
        headers = {'Authorization': self.auth_token}

        response = self._send_request(method, url, headers=headers, json=json, params=params)

        # this retries the request if authentication is revoked
        if response.status_code == 401 or self._is_invalid_token_response(response):
            self._login()
            headers = {'Authorization': self.auth_token}

            response = self._send_request(method, url, headers=headers, json=json, params=params)

        return response

    def _send_request(self, method, url, params=None, json=None, headers=None, retry=False):
        try:
            response = requests.request(method, url, params=params, json=json, headers=headers,
                                        timeout=self.DEFAULT_REQUEST_TIMEOUT_IN_SECONDS)
        except requests.exceptions.Timeout:
            raise ResponsysClientError('There was a timeout error sending a request to Responsys.'
                                    'Method: {}, URL: {}'.format(method, url))
        except Exception:
            raise ResponsysClientError('There was an unknown error sending a request to Responsys.'
                                    'Method: {}, URL: {}'.format(method, url))

        if not retry and response.status_code == 429:
            logger.info('This Responsys API call got rate limited: {} {}.'.format(method, url))
            time.sleep(self.RESPONSYS_RATE_LIMIT_WAITING_PERIOD_IN_SECONDS)

            response = self._send_request(method, url, headers=headers, retry=True)

        return response

    def send_merge_request(self, path, member_field_names, member_records):
        if len(member_records) > self.RECORD_PROCESS_LIMIT_QUANTITY:
            raise ResponsysClientError('A max of 200 members may be created or updated at one time.')

        json = {
                "recordData": {
                    "fieldNames": member_field_names,
                    "records": member_records,
                    "mapTemplateName": None
                },
                "mergeRule": {
                    "htmlValue": "H",
                    "optinValue": "I",
                    "textValue": "T",
                    "insertOnNoMatch": True,
                    "updateOnMatch": "REPLACE_ALL",
                    "matchColumnName1": "customer_id_",
                    "matchColumnName2": None,
                    "matchOperator": "NONE",
                    "optoutValue": "O",
                    "rejectRecordIfChannelEmpty": None,
                    "defaultPermissionStatus": "OPTIN"<commit_msg>remove logger<commit_after>import time
from datetime import datetime
from datetime import timedelta
from urlparse import urljoin

import pytz
import requests

from exceptions import ResponsysClientError
from utils import convert_to_list_of_dicts
from utils import convert_to_table_structure


class ResponsysClient(object):

    AUTH_TOKEN_REFRESH_THRESHOLD = timedelta(hours=1)
    AUTH_PATH = '/rest/api/v1.1/auth/token'
    RESPONSYS_RATE_LIMIT_WAITING_PERIOD_IN_SECONDS = 60
    RESPONSYS_INVALID_TOKEN_RESPONSE_DETAIL = 'Not a valid authentication token'
    DEFAULT_REQUEST_TIMEOUT_IN_SECONDS = 60
    RECORD_PROCESS_LIMIT_QUANTITY = 200

    def __init__(self, username=None,
                 password=<PASSWORD>,
                 login_url=None,
                 profile_list=None,
                 folder_name=None,
                 rental_supp_table=None):

        self.username = username
        self.password = password
        self.login_url = login_url
        self.issued_url = None
        self.auth_token = None
        self.refresh_timestamp = None
        self.profile_list = profile_list
        self.folder = folder_name
        self.rental_supp_table = rental_supp_table

    def get_profile_contact_lists(self):
        method = 'GET'
        path = '/rest/api/v1.1/lists'

        response = self.send_request(method, path)

        self._check_for_valid_response(method, path, response.status_code, response.text)

        return response.json()

    def get_extension_tables_lists(self):
        method = 'GET'
        path = '/rest/api/v1.1/lists/{}/listExtensions'.format(self.profile_list)

        response = self.send_request(method, path)

        self._check_for_valid_response(method, path, response.status_code, response.text)

        return response.json()

    def create_or_update_contact_list_members(self, contact_dicts):
        member_field_names, member_records = convert_to_table_structure(contact_dicts)

        path = '/rest/api/v1.1/lists/{}/members'.format(self.profile_list)

        response = self.send_merge_request(path, member_field_names, member_records)

        self._check_for_valid_response(response.request.method, path, response.status_code,
                                       response.text)

        return response.json()

    def create_or_update_extension_table_members(self, table_name, data_dicts):
        member_field_names, member_records = convert_to_table_structure(data_dicts)

        path = ('/rest/api/v1.1/lists/{}/listExtensions/{}/members'
                .format(self.profile_list, table_name))

        response = self.send_ext_table_merge_request(path, member_field_names, member_records)

        self._check_for_valid_response(response.request.method,
                                       path, response.status_code, response.text)

        return response.json()

    def create_or_update_supplemental_table_members(self, table_name, data_dicts):
        member_field_names, member_records = convert_to_table_structure(data_dicts)

        path = ('/rest/api/v1.1/folders/{}/suppData/{}/members'
                .format(self.folder, table_name))

        response = self.send_supp_table_merge_request(path, member_field_names, member_records)

        self._check_for_valid_response(response.request.method, path, response.status_code,
                                       response.text)

        return response.json()

    def get_contact_list_table_member(self, user_id):
        method = 'GET'
        path = '/rest/api/v1.1/lists/{}/members/'.format(self.profile_list)
        query_by_customer_id_params = {
            'qa': 'c',
            'id': str(user_id),
            'fs': 'all'
        }

        response = self.send_request(method, path, params=query_by_customer_id_params)

        self._check_for_valid_response(method, path, response.status_code, response.text)

        members = self._parse_members(response)

        return members[0]

    def get_extension_table_member(self, table_name, user_id):
        method = 'GET'
        path = ('/rest/api/v1.1/lists/{}/listExtensions/{}/members'
                .format(self.profile_list, table_name))
        query_by_customer_id_params = {
            'qa': 'c',
            'id': str(user_id),
            'fs': 'all'
        }

        response = self.send_request(method, path, params=query_by_customer_id_params)

        self._check_for_valid_response(method, path, response.status_code, response.text)

        members = self._parse_members(response)

        return members[0]

    def delete_contact_list_member(self, user_id):
        member = self.get_contact_list_table_member(user_id)

        method = 'DELETE'
        path = '/rest/api/v1.1/lists/{}/members/{}'.format(self.profile_list, member['RIID_'])

        response = self.send_request(method, path)

        self._check_for_valid_response(method, path, response.status_code, response.text)

    def delete_extension_table_member(self, table_name, user_id):
        member = self.get_extension_table_member(table_name, user_id)

        method = 'DELETE'
        path = ('/rest/api/v1.1/lists/{}/listExtensions/{}/members/{}'
                .format(self.profile_list, table_name, member['RIID_']))

        response = self.send_request(method, path)

        self._check_for_valid_response(method, path, response.status_code, response.text)

    @staticmethod
    def _parse_members(response):
        parsed_response = response.json()

        field_names = parsed_response['recordData']['fieldNames']
        records = parsed_response['recordData']['records']

        members = convert_to_list_of_dicts(field_names, records)

        return members

    def send_request(self, method, path, params=None, json=None):
        self._get_access_items()

        url = urljoin(self.issued_url, path)
        headers = {'Authorization': self.auth_token}

        response = self._send_request(method, url, headers=headers, json=json, params=params)

        # this retries the request if authentication is revoked
        if response.status_code == 401 or self._is_invalid_token_response(response):
            self._login()
            headers = {'Authorization': self.auth_token}

            response = self._send_request(method, url, headers=headers, json=json, params=params)

        return response

    def _send_request(self, method, url, params=None, json=None, headers=None, retry=False):
        try:
            response = requests.request(method, url, params=params, json=json, headers=headers,
                                        timeout=self.DEFAULT_REQUEST_TIMEOUT_IN_SECONDS)
        except requests.exceptions.Timeout:
            raise ResponsysClientError('There was a timeout error sending a request to Responsys.'
                                    'Method: {}, URL: {}'.format(method, url))
        except Exception:
            raise ResponsysClientError('There was an unknown error sending a request to Responsys.'
                                    'Method: {}, URL: {}'.format(method, url))

        if not retry and response.status_code == 429:
            time.sleep(self.RESPONSYS_RATE_LIMIT_WAITING_PERIOD_IN_SECONDS)

            response = self._send_request(method, url, headers=headers, retry=True)

        return response

    def send_merge_request(self, path, member_field_names, member_records):
        if len(member_records) > self.RECORD_PROCESS_LIMIT_QUANTITY:
            raise ResponsysClientError('A max of 200 members may be created or updated at one time.')

        json = {
                "recordData": {
                    "fieldNames": member_field_names,
                    "records": member_records,
                    "mapTemplateName": None
                },
                "mergeRule": {
                    "htmlValue": "H",
                    "optinValue": "I",
                    "textValue": "T",
                    "insertOnNoMatch": True,
                    "updateOnMatch": "REPLACE_ALL",
                    "matchColumnName1": "customer_id_",
                    "matchColumnName2": None,
                    "matchOperator": "NONE",
                    "optoutValue": "O",
                    "rejectRecordIfChannelEmpty": None,
                    "defaultPermissionStatus": "OPTIN"
{'fraction_non_alphanumeric': 0.09567293345270068, 'fraction_numerical': 0.0048940614741868095, 'mean_word_length': 2.5409974640743873, 'pattern_counts': {'":': 32, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3168343', 'n_tokens_mistral': 4884, 'n_tokens_neox': 4672, 'n_words': 877}