<commit_before>"""
The influenceexplorer module provides API wrappers for the data provided on 
InfluenceExplorer.com.
"""


import urllib2
import urllib
try:
    import json
except ImportError:
    import simplejson as json
    
from transparencydata import DEFAULT_URL


# defaults of None don't mean that there is not default or no limit--
# it means that no parameter will be sent to the server, and the server
# will use its own default.
DEFAULT_LIMIT = None

DEFAULT_CYCLE = "-1" # -1 will return career totals.


class InfluenceExplorer(object):
    
    """
    Provides wrappers to the Influence Explorer APIs.
    
    Methods are divided into four categories. Entity methods can be accessed
    through the ``entities`` member, Politician methods through ``pol``, Individual
    methods through ``indiv`` and Organization through ``org``. For example::
    
        api = InfluenceExplorer(<your-key-here>)
        boehner_id = api.entities.search('<NAME>')[1]['id]
        print api.pol.industries(boehner_id)
    """

    def __init__(self, api_key, base_url=DEFAULT_URL):
        """
        Create an API wrapper. 
        
        API keys can be obtained from http://services.sunlightlabs.com.
        """
        
        self.base_url = base_url if base_url[-1] == '/' else base_url + '/'
        self.api_key = api_key
        self.entities = Entities(self)
        self.pol = Politician(self)
        self.indiv = Individual(self)
        self.org = Organization(self)


    def _get_url_json(self, path, cycle=None, limit=None, **params):
        """ Low level call that just adds the API key, retrieves the URL and parses the JSON. """

        if cycle:
            params.update({'cycle': cycle})
        if limit:
            params.update({'limit': limit})
        
        params.update({'apikey': self.api_key})

        fp = urllib2.urlopen(self.base_url + path + '?' + urllib.urlencode(params))

        return json.loads(fp.read())


class SubAPI(object):
    def __init__(self, main_api):
        self._get_url_json = main_api._get_url_json
        

class Entities(SubAPI):
    """
    Methods related to searching, listing and ranking entities.
    
    Accessed as ``InfluenceExplorer.entities``.
    """
    
    def search(self, query):
        """
        Return entities with names matching the given query.
        
        Query terms are space separated. Matches must contain all terms
        from the query.
        """
        
        return self._get_url_json('entities.json', search=query.encode('ascii', 'ignore'))

    _camp_fin_markers = ['contributor_count', 'recipient_count', 'independent_expenditure_amount', 'fec_summary_count']
    _lobbying_markers = ['lobbying_count']
    _spending_markers = ['grant_count', 'loan_count', 'contract_count']
    _earmark_markers = ['earmark_count']
    _contractor_misconduct_markers = ['contractor_misconduct_count']
    _epa_echo_markers = ['epa_actions_count']
    _regulations_markers = ['regs_docket_count', 'regs_submitted_docket_count']
    _faca_markers = ['faca_committee_count', 'faca_member_count']

    def metadata(self, entity_id):
        
        """Return all available metadata for the given entity."""
        
        results = self._get_url_json('entities/%s.json' % entity_id)

        results['years'] = self._entity_years(results['totals'], self._camp_fin_markers + self._lobbying_markers + self._spending_markers)
        results['camp_fin_years'] = self._entity_years(results['totals'], self._camp_fin_markers)
        results['lobbying_years'] = self._entity_years(results['totals'], self._lobbying_markers)
        results['spending_years'] = self._entity_years(results['totals'], self._spending_markers)
        results['earmark_years'] = self._entity_years(results['totals'], self._earmark_markers)
        results['contractor_misconduct_years'] = self._entity_years(results['totals'], self._contractor_misconduct_markers)
        results['epa_echo_years'] = self._entity_years(results['totals'], self._epa_echo_markers)
        results['regulations_years'] = self._entity_years(results['totals'], self._regulations_markers)
        results['faca_years'] = self._entity_years(results['totals'], self._faca_markers)

        return results

    def _entity_years(self, totals, keys):
        years = [year for (year, values) in totals.items() if any([v for (k,v) in values.items() if k in keys]) and year != "-1"]
        years.sort()
        if years:
            return dict(start=years[0], end=years[-1])
        else:
            return {}

    def id_lookup(self, namespace, id):
        """
        Return the Influence Explorer entity ID based on a 3rd party ID.
        
        Valid namespaces include:
        
        * ``urn:crp:individual`` -- CRP's contributor or lobbyist ID
        * ``urn:crp:organization`` -- CRP's organization ID
        * ``urn:crp:recipient`` -- CRP's candidate ID
        * ``urn:crp:industry`` -- CRP's 3-letter category order
        * ``urn:crp:subindustry`` -- CRP's 5-letter category code
        * ``urn:nimsp:subindustry`` -- 5-letter category code added by NIMSP
        * ``urn:nimsp:organization`` -- NIMSP's organization ID
        * ``urn:nimsp:recipient`` -- NIMSP's candidate ID
        * ``urn:sunlight:lobbyist_registration_tracker_url`` -- URL of Sunlight's lobbyist registration tracker page
        """
        return self._get_url_json('entities/id_lookup.json', namespace=namespace, id=id)


    def count(self, type=None):
        """ Return the total count of entities. """
        params = {'count': 1}
        if type:
            params['type'] = type
        return int(self._get_url_json('entities/list.json', **params)['count'])


    def list(self, start, end, type=None):
        """ List all entities. """
        params = {'start': start, 'end': end}
        if type:
            params['type'] = type
        return self._get_url_json('entities/list.json', **params)


    # top n lists
    def top_n_individuals(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top individuals, by amount contributed. """
        return self._get_url_json('aggregates/indivs/top_%s.json' % limit, cycle)

    def top_n_individual_donors_to_democrats(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top individuals, by amount contributed to Democrats. """
        return self._get_url_json('aggregates/indivs/party/D/top_%s.json' % limit, cycle)

    def top_n_individual_donors_to_republicans(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top individuals, by amount contributed to Republicans. """
        return self._get_url_json('aggregates/indivs/party/R/top_%s.json' % limit, cycle)

    def top_n_organizations(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top organizations, by amount contributed. """
        return self._get_url_json('aggregates/orgs/top_%s.json' % limit, cycle)

    def top_n_politicians(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top politicians, by amount received. """
        return self._get_url_json('aggregates/pols/top_%s.json' % limit, cycle)

    def top_n_industries(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top industries, by amount contributed. """
        return self._get_url_json('aggregates/industries/top_%s.json' % limit, cycle)

    def candidates_by_location(self, location, cycle=DEFAULT_CYCLE):
        """ Internal use only. Not maintained. """
        return self._get_url_json('entities/race/%s.json' % location, cycle)

    def election_districts(self, cycle=DEFAULT_CYCLE):
        """ Internal use only. Not maintained. """
        return self._get_url_json('entities/race/districts.json', cycle)

    def bundles(self, entity_id, cycle=DEFAULT_CYCLE):
        """ Return any bundling data for the entity. """
        return self._get_url_json('aggregates/pol/{0}/bundles.json'.format(entity_id), cycle)

    def top_n_lobbyist_bundlers(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return top lobbyist bundlers. """
        return self._get_url_json('aggregates/indivs/lobbyist_bundlers/top_{0}.json'.format(limit), cycle)<commit_msg>[#1073] Changing indiv donors to party method names<commit_after>"""
The influenceexplorer module provides API wrappers for the data provided on 
InfluenceExplorer.com.
"""


import urllib2
import urllib
try:
    import json
except ImportError:
    import simplejson as json
    
from transparencydata import DEFAULT_URL


# defaults of None don't mean that there is not default or no limit--
# it means that no parameter will be sent to the server, and the server
# will use its own default.
DEFAULT_LIMIT = None

DEFAULT_CYCLE = "-1" # -1 will return career totals.


class InfluenceExplorer(object):
    
    """
    Provides wrappers to the Influence Explorer APIs.
    
    Methods are divided into four categories. Entity methods can be accessed
    through the ``entities`` member, Politician methods through ``pol``, Individual
    methods through ``indiv`` and Organization through ``org``. For example::
    
        api = InfluenceExplorer(<your-key-here>)
        boehner_id = api.entities.search('<NAME>')[1]['id]
        print api.pol.industries(boehner_id)
    """

    def __init__(self, api_key, base_url=DEFAULT_URL):
        """
        Create an API wrapper. 
        
        API keys can be obtained from http://services.sunlightlabs.com.
        """
        
        self.base_url = base_url if base_url[-1] == '/' else base_url + '/'
        self.api_key = api_key
        self.entities = Entities(self)
        self.pol = Politician(self)
        self.indiv = Individual(self)
        self.org = Organization(self)


    def _get_url_json(self, path, cycle=None, limit=None, **params):
        """ Low level call that just adds the API key, retrieves the URL and parses the JSON. """

        if cycle:
            params.update({'cycle': cycle})
        if limit:
            params.update({'limit': limit})
        
        params.update({'apikey': self.api_key})

        fp = urllib2.urlopen(self.base_url + path + '?' + urllib.urlencode(params))

        return json.loads(fp.read())


class SubAPI(object):
    def __init__(self, main_api):
        self._get_url_json = main_api._get_url_json
        

class Entities(SubAPI):
    """
    Methods related to searching, listing and ranking entities.
    
    Accessed as ``InfluenceExplorer.entities``.
    """
    
    def search(self, query):
        """
        Return entities with names matching the given query.
        
        Query terms are space separated. Matches must contain all terms
        from the query.
        """
        
        return self._get_url_json('entities.json', search=query.encode('ascii', 'ignore'))

    _camp_fin_markers = ['contributor_count', 'recipient_count', 'independent_expenditure_amount', 'fec_summary_count']
    _lobbying_markers = ['lobbying_count']
    _spending_markers = ['grant_count', 'loan_count', 'contract_count']
    _earmark_markers = ['earmark_count']
    _contractor_misconduct_markers = ['contractor_misconduct_count']
    _epa_echo_markers = ['epa_actions_count']
    _regulations_markers = ['regs_docket_count', 'regs_submitted_docket_count']
    _faca_markers = ['faca_committee_count', 'faca_member_count']

    def metadata(self, entity_id):
        
        """Return all available metadata for the given entity."""
        
        results = self._get_url_json('entities/%s.json' % entity_id)

        results['years'] = self._entity_years(results['totals'], self._camp_fin_markers + self._lobbying_markers + self._spending_markers)
        results['camp_fin_years'] = self._entity_years(results['totals'], self._camp_fin_markers)
        results['lobbying_years'] = self._entity_years(results['totals'], self._lobbying_markers)
        results['spending_years'] = self._entity_years(results['totals'], self._spending_markers)
        results['earmark_years'] = self._entity_years(results['totals'], self._earmark_markers)
        results['contractor_misconduct_years'] = self._entity_years(results['totals'], self._contractor_misconduct_markers)
        results['epa_echo_years'] = self._entity_years(results['totals'], self._epa_echo_markers)
        results['regulations_years'] = self._entity_years(results['totals'], self._regulations_markers)
        results['faca_years'] = self._entity_years(results['totals'], self._faca_markers)

        return results

    def _entity_years(self, totals, keys):
        years = [year for (year, values) in totals.items() if any([v for (k,v) in values.items() if k in keys]) and year != "-1"]
        years.sort()
        if years:
            return dict(start=years[0], end=years[-1])
        else:
            return {}

    def id_lookup(self, namespace, id):
        """
        Return the Influence Explorer entity ID based on a 3rd party ID.
        
        Valid namespaces include:
        
        * ``urn:crp:individual`` -- CRP's contributor or lobbyist ID
        * ``urn:crp:organization`` -- CRP's organization ID
        * ``urn:crp:recipient`` -- CRP's candidate ID
        * ``urn:crp:industry`` -- CRP's 3-letter category order
        * ``urn:crp:subindustry`` -- CRP's 5-letter category code
        * ``urn:nimsp:subindustry`` -- 5-letter category code added by NIMSP
        * ``urn:nimsp:organization`` -- NIMSP's organization ID
        * ``urn:nimsp:recipient`` -- NIMSP's candidate ID
        * ``urn:sunlight:lobbyist_registration_tracker_url`` -- URL of Sunlight's lobbyist registration tracker page
        """
        return self._get_url_json('entities/id_lookup.json', namespace=namespace, id=id)


    def count(self, type=None):
        """ Return the total count of entities. """
        params = {'count': 1}
        if type:
            params['type'] = type
        return int(self._get_url_json('entities/list.json', **params)['count'])


    def list(self, start, end, type=None):
        """ List all entities. """
        params = {'start': start, 'end': end}
        if type:
            params['type'] = type
        return self._get_url_json('entities/list.json', **params)


    # top n lists
    def top_n_individuals(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top individuals, by amount contributed. """
        return self._get_url_json('aggregates/indivs/top_%s.json' % limit, cycle)

    def top_n_indiv_democratic_donors(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top individuals, by amount contributed to Democrats. """
        return self._get_url_json('aggregates/indivs/party/D/top_%s.json' % limit, cycle)

    def top_n_indiv_republican_donors(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top individuals, by amount contributed to Republicans. """
        return self._get_url_json('aggregates/indivs/party/R/top_%s.json' % limit, cycle)

    def top_n_organizations(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top organizations, by amount contributed. """
        return self._get_url_json('aggregates/orgs/top_%s.json' % limit, cycle)

    def top_n_politicians(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top politicians, by amount received. """
        return self._get_url_json('aggregates/pols/top_%s.json' % limit, cycle)

    def top_n_industries(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return the top industries, by amount contributed. """
        return self._get_url_json('aggregates/industries/top_%s.json' % limit, cycle)

    def candidates_by_location(self, location, cycle=DEFAULT_CYCLE):
        """ Internal use only. Not maintained. """
        return self._get_url_json('entities/race/%s.json' % location, cycle)

    def election_districts(self, cycle=DEFAULT_CYCLE):
        """ Internal use only. Not maintained. """
        return self._get_url_json('entities/race/districts.json', cycle)

    def bundles(self, entity_id, cycle=DEFAULT_CYCLE):
        """ Return any bundling data for the entity. """
        return self._get_url_json('aggregates/pol/{0}/bundles.json'.format(entity_id), cycle)

    def top_n_lobbyist_bundlers(self, cycle=DEFAULT_CYCLE, limit=DEFAULT_LIMIT):
        """ Return top lobbyist bundlers. """
        return self._get_url_json('aggregates/indivs/lobbyist_bundlers/top_{0}.json'.format(limit), cycle)
{'fraction_non_alphanumeric': 0.1186122547832405, 'fraction_numerical': 0.0021797045289416324, 'mean_word_length': 3.250386001029336, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2089244', 'n_tokens_mistral': 5132, 'n_tokens_neox': 4916, 'n_words': 1180}