def get_data(contract_type, award_type, now, sess, sub_tier_list, county_by_name, county_by_code, state_code_list, country_list, exec_comp_dict, last_run=None, threaded=False, start_date=None, end_date=None, metrics=None, specific_params=None):
    " Get the data from the atom feed based on contract/award type and the last time the script was run.\n\n        Args:\n            contract_type: a string indicating whether the atom feed being checked is 'award' or 'IDV'\n            award_type: a string indicating what the award type of the feed being checked is\n            now: a timestamp indicating the time to set the updated_at to\n            sess: the database connection\n            sub_tier_list: a dictionary containing all the sub tier agency information keyed by sub tier agency code\n            county_by_name: a dictionary containing all county codes, keyed by state and county name\n            county_by_code: a dictionary containing all county names, keyed by state and county code\n            state_code_list: a dictionary containing all state names, keyed by state code\n            country_list: a dictionary containing all country names, keyed by country code\n            exec_comp_dict: a dictionary containing all the data for Executive Compensation data keyed by DUNS number\n            last_run: a date indicating the last time the pull was run\n            threaded: a boolean indicating whether the process is running as a thread or not\n            start_date: a date indicating the first date to pull from (must be provided with end_date)\n            end_date: a date indicating the last date to pull from (must be provided with start_date)\n            metrics: a dictionary to gather metrics for the script in\n            specific_params: a string containing a specific set of params to run the query with (used for outside\n                scripts that need to run a data load)\n    "
    if (not metrics):
        metrics = {}
    data = []
    yesterday = (now - datetime.timedelta(days=1))
    utcnow = datetime.datetime.utcnow()
    if specific_params:
        params = specific_params
    elif (not last_run):
        params = (('SIGNED_DATE:[2016/10/01,' + yesterday.strftime('%Y/%m/%d')) + '] ')
        metrics['start_date'] = '2016/10/01'
        metrics['end_date'] = yesterday.strftime('%Y/%m/%d')
    else:
        last_run_date = (last_run - relativedelta(days=1))
        params = (((('LAST_MOD_DATE:[' + last_run_date.strftime('%Y/%m/%d')) + ',') + yesterday.strftime('%Y/%m/%d')) + '] ')
        metrics['start_date'] = last_run_date.strftime('%Y/%m/%d')
        metrics['end_date'] = yesterday.strftime('%Y/%m/%d')
        if (start_date and end_date):
            params = (((('LAST_MOD_DATE:[' + start_date) + ',') + end_date) + '] ')
            metrics['start_date'] = start_date
            metrics['end_date'] = end_date
    base_url = ((((((feed_url + params) + 'CONTRACT_TYPE:"') + contract_type.upper()) + '" AWARD_TYPE:"') + award_type) + '"')
    logger.info('Starting get feed: %s', base_url)
    total_expected_records = get_total_expected_records(base_url)
    logger.info('{} record(s) expected from this feed'.format(total_expected_records))
    entries_processed = 0
    while True:

        async def atom_async_get(entries_already_processed, total_expected_records):
            response_list = []
            loop = asyncio.get_event_loop()
            requests_at_once = MAX_REQUESTS_AT_ONCE
            if ((total_expected_records - entries_already_processed) < (MAX_REQUESTS_AT_ONCE * MAX_ENTRIES)):
                requests_at_once = (math.ceil(((total_expected_records - entries_already_processed) / MAX_ENTRIES)) + 1)
            futures = [loop.run_in_executor(None, get_with_exception_hand, ((base_url + '&start=') + str((entries_already_processed + (start_offset * MAX_ENTRIES)))), (total_expected_records > (entries_already_processed + (start_offset * MAX_ENTRIES)))) for start_offset in range(requests_at_once)]
            for response in (await asyncio.gather(*futures)):
                response_list.append(response.text)
                pass
            return response_list
        loop = asyncio.get_event_loop()
        full_response = loop.run_until_complete(atom_async_get(entries_processed, total_expected_records))
        for next_resp in full_response:
            response_dict = xmltodict.parse(next_resp, process_namespaces=True, namespaces=FPDS_NAMESPACES)
            try:
                entries_per_response = list_data(response_dict['feed']['entry'])
            except KeyError:
                continue
            if (last_run or specific_params):
                for entry in entries_per_response:
                    data.append(entry)
                    entries_processed += 1
            else:
                data.extend(create_processed_data_list(entries_per_response, contract_type, sess, sub_tier_list, county_by_name, county_by_code, state_code_list, country_list, exec_comp_dict))
                entries_processed += len(entries_per_response)
        if (entries_processed > total_expected_records):
            for next_resp in full_response:
                response_dict = xmltodict.parse(next_resp, process_namespaces=True, namespaces=FPDS_NAMESPACES)
                try:
                    list_data(response_dict['feed']['entry'])
                except KeyError:
                    logger.info(response_dict)
                    continue
            raise Exception('Total number of expected records has changed\nExpected: {}\nRetrieved so far: {}'.format(total_expected_records, len(data)))
        if data:
            logger.info('Retrieved %s lines of get %s: %s feed, writing next %s to DB', entries_processed, contract_type, award_type, len(data))
            if (last_run or specific_params):
                process_and_add(data, contract_type, sess, sub_tier_list, county_by_name, county_by_code, state_code_list, country_list, exec_comp_dict, utcnow, threaded)
            else:
                add_processed_data_list(data, sess)
            logger.info('Successfully inserted %s lines of get %s: %s feed, continuing feed retrieval', len(data), contract_type, award_type)
        if (len(data) < (MAX_ENTRIES * MAX_REQUESTS_AT_ONCE)):
            if (entries_processed != total_expected_records):
                raise Exception('Records retrieved != Total expected records\nExpected: {}\nRetrieved: {}'.format(total_expected_records, entries_processed))
            else:
                if ('records_received' not in metrics):
                    metrics['records_received'] = total_expected_records
                else:
                    metrics['records_received'] += total_expected_records
                break
        else:
            data = []
    logger.info('Total entries in %s: %s feed: %s', contract_type, award_type, entries_processed)
    logger.info('Processed %s: %s data', contract_type, award_type)
Get the data from the atom feed based on contract/award type and the last time the script was run.

Args:
    contract_type: a string indicating whether the atom feed being checked is 'award' or 'IDV'
    award_type: a string indicating what the award type of the feed being checked is
    now: a timestamp indicating the time to set the updated_at to
    sess: the database connection
    sub_tier_list: a dictionary containing all the sub tier agency information keyed by sub tier agency code
    county_by_name: a dictionary containing all county codes, keyed by state and county name
    county_by_code: a dictionary containing all county names, keyed by state and county code
    state_code_list: a dictionary containing all state names, keyed by state code
    country_list: a dictionary containing all country names, keyed by country code
    exec_comp_dict: a dictionary containing all the data for Executive Compensation data keyed by DUNS number
    last_run: a date indicating the last time the pull was run
    threaded: a boolean indicating whether the process is running as a thread or not
    start_date: a date indicating the first date to pull from (must be provided with end_date)
    end_date: a date indicating the last date to pull from (must be provided with start_date)
    metrics: a dictionary to gather metrics for the script in
    specific_params: a string containing a specific set of params to run the query with (used for outside
        scripts that need to run a data load)
def get_data(contract_type, award_type, now, sess, sub_tier_list, county_by_name, county_by_code, state_code_list, country_list, exec_comp_dict, last_run=None, threaded=False, start_date=None, end_date=None, metrics=None, specific_params=None):
    " Get the data from the atom feed based on contract/award type and the last time the script was run.\n\n        Args:\n            contract_type: a string indicating whether the atom feed being checked is 'award' or 'IDV'\n            award_type: a string indicating what the award type of the feed being checked is\n            now: a timestamp indicating the time to set the updated_at to\n            sess: the database connection\n            sub_tier_list: a dictionary containing all the sub tier agency information keyed by sub tier agency code\n            county_by_name: a dictionary containing all county codes, keyed by state and county name\n            county_by_code: a dictionary containing all county names, keyed by state and county code\n            state_code_list: a dictionary containing all state names, keyed by state code\n            country_list: a dictionary containing all country names, keyed by country code\n            exec_comp_dict: a dictionary containing all the data for Executive Compensation data keyed by DUNS number\n            last_run: a date indicating the last time the pull was run\n            threaded: a boolean indicating whether the process is running as a thread or not\n            start_date: a date indicating the first date to pull from (must be provided with end_date)\n            end_date: a date indicating the last date to pull from (must be provided with start_date)\n            metrics: a dictionary to gather metrics for the script in\n            specific_params: a string containing a specific set of params to run the query with (used for outside\n                scripts that need to run a data load)\n    "
    if (not metrics):
        metrics = {}
    data = []
    yesterday = (now - datetime.timedelta(days=1))
    utcnow = datetime.datetime.utcnow()
    if specific_params:
        params = specific_params
    elif (not last_run):
        params = (('SIGNED_DATE:[2016/10/01,' + yesterday.strftime('%Y/%m/%d')) + '] ')
        metrics['start_date'] = '2016/10/01'
        metrics['end_date'] = yesterday.strftime('%Y/%m/%d')
    else:
        last_run_date = (last_run - relativedelta(days=1))
        params = (((('LAST_MOD_DATE:[' + last_run_date.strftime('%Y/%m/%d')) + ',') + yesterday.strftime('%Y/%m/%d')) + '] ')
        metrics['start_date'] = last_run_date.strftime('%Y/%m/%d')
        metrics['end_date'] = yesterday.strftime('%Y/%m/%d')
        if (start_date and end_date):
            params = (((('LAST_MOD_DATE:[' + start_date) + ',') + end_date) + '] ')
            metrics['start_date'] = start_date
            metrics['end_date'] = end_date
    base_url = ((((((feed_url + params) + 'CONTRACT_TYPE:"') + contract_type.upper()) + '" AWARD_TYPE:"') + award_type) + '"')
    logger.info('Starting get feed: %s', base_url)
    total_expected_records = get_total_expected_records(base_url)
    logger.info('{} record(s) expected from this feed'.format(total_expected_records))
    entries_processed = 0
    while True:

        async def atom_async_get(entries_already_processed, total_expected_records):
            response_list = []
            loop = asyncio.get_event_loop()
            requests_at_once = MAX_REQUESTS_AT_ONCE
            if ((total_expected_records - entries_already_processed) < (MAX_REQUESTS_AT_ONCE * MAX_ENTRIES)):
                requests_at_once = (math.ceil(((total_expected_records - entries_already_processed) / MAX_ENTRIES)) + 1)
            futures = [loop.run_in_executor(None, get_with_exception_hand, ((base_url + '&start=') + str((entries_already_processed + (start_offset * MAX_ENTRIES)))), (total_expected_records > (entries_already_processed + (start_offset * MAX_ENTRIES)))) for start_offset in range(requests_at_once)]
            for response in (await asyncio.gather(*futures)):
                response_list.append(response.text)
                pass
            return response_list
        loop = asyncio.get_event_loop()
        full_response = loop.run_until_complete(atom_async_get(entries_processed, total_expected_records))
        for next_resp in full_response:
            response_dict = xmltodict.parse(next_resp, process_namespaces=True, namespaces=FPDS_NAMESPACES)
            try:
                entries_per_response = list_data(response_dict['feed']['entry'])
            except KeyError:
                continue
            if (last_run or specific_params):
                for entry in entries_per_response:
                    data.append(entry)
                    entries_processed += 1
            else:
                data.extend(create_processed_data_list(entries_per_response, contract_type, sess, sub_tier_list, county_by_name, county_by_code, state_code_list, country_list, exec_comp_dict))
                entries_processed += len(entries_per_response)
        if (entries_processed > total_expected_records):
            for next_resp in full_response:
                response_dict = xmltodict.parse(next_resp, process_namespaces=True, namespaces=FPDS_NAMESPACES)
                try:
                    list_data(response_dict['feed']['entry'])
                except KeyError:
                    logger.info(response_dict)
                    continue
            raise Exception('Total number of expected records has changed\nExpected: {}\nRetrieved so far: {}'.format(total_expected_records, len(data)))
        if data:
            logger.info('Retrieved %s lines of get %s: %s feed, writing next %s to DB', entries_processed, contract_type, award_type, len(data))
            if (last_run or specific_params):
                process_and_add(data, contract_type, sess, sub_tier_list, county_by_name, county_by_code, state_code_list, country_list, exec_comp_dict, utcnow, threaded)
            else:
                add_processed_data_list(data, sess)
            logger.info('Successfully inserted %s lines of get %s: %s feed, continuing feed retrieval', len(data), contract_type, award_type)
        if (len(data) < (MAX_ENTRIES * MAX_REQUESTS_AT_ONCE)):
            if (entries_processed != total_expected_records):
                raise Exception('Records retrieved != Total expected records\nExpected: {}\nRetrieved: {}'.format(total_expected_records, entries_processed))
            else:
                if ('records_received' not in metrics):
                    metrics['records_received'] = total_expected_records
                else:
                    metrics['records_received'] += total_expected_records
                break
        else:
            data = []
    logger.info('Total entries in %s: %s feed: %s', contract_type, award_type, entries_processed)
    logger.info('Processed %s: %s data', contract_type, award_type)