<commit_before>import requests, requests.utils
import time, re
import pdb

class PyMsCognitiveException(Exception):
    pass

class PyMsCognitiveSearch(object):
    """
    Shell class for the individual searches
    """
    def __init__(self, api_key, query, query_url, safe=False):
        self.api_key = api_key
        self.safe = safe
        self.current_offset = 0
        self.query = query
        self.QUERY_URL = query_url

    def get_json_results(self, response):
        '''
        Parses the request result and returns the JSON object. Handles all errors.
        '''
        try:
            # return the proper JSON object, or error code if request didn't go through.
            json_results = response.json()
            if response.status_code in [401, 403]: #401 is invalid key, 403 is out of monthly quota.
                raise PyMsCognitiveWebSearchException("CODE {code}: {message}".format(code=response.status_code,message=json_results["message"]) )
            elif response.status_code in [429]:
                message = json_results['message']
                try:
                    # extract time out seconds from response
                    timeout = int(re.search('in (.+?) seconds', message).group(1)) + 1
                    print ("CODE 429, sleeping for {timeout} seconds").format(timeout=str(timeout))<commit_msg>saving most_recent_json<commit_after>import requests, requests.utils
import time, re
import pdb

class PyMsCognitiveException(Exception):
    pass

class PyMsCognitiveSearch(object):
    """
    Shell class for the individual searches
    """
    def __init__(self, api_key, query, query_url, safe=False):
        self.api_key = api_key
        self.safe = safe
        self.current_offset = 0
        self.query = query
        self.QUERY_URL = query_url
        self.most_recent_json = None

    def get_json_results(self, response):
        '''
        Parses the request result and returns the JSON object. Handles all errors.
        '''
        try:
            # return the proper JSON object, or error code if request didn't go through.
            self.most_recent_json = response.json()
            json_results = response.json()
            if response.status_code in [401, 403]: #401 is invalid key, 403 is out of monthly quota.
                raise PyMsCognitiveWebSearchException("CODE {code}: {message}".format(code=response.status_code,message=json_results["message"]) )
            elif response.status_code in [429]:
                message = json_results['message']
                try:
                    # extract time out seconds from response
                    timeout = int(re.search('in (.+?) seconds', message).group(1)) + 1
                    print ("CODE 429, sleeping for {timeout} seconds").format(timeout=str(timeout))
{'fraction_non_alphanumeric': 0.09291562833748665, 'fraction_numerical': 0.014951940192239232, 'mean_word_length': 2.3814681107099878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2014384', 'n_tokens_mistral': 789, 'n_tokens_neox': 722, 'n_words': 224}