<commit_before>import json

import requests
from IPStreet.error import APIConnectionError, SendError

from IPStreet import query


class Client:
    """Basic IP Street API Client Object"""

    def __init__(self, apikey, api_version):

        self.debug = True

        # credentials
        self.apikey = apikey
        self.headers = {'x-api-key': self.apikey}

        # create most up-to-date endpoints
        self.host = 'https://api.ipstreet.com'
        self.api_version = "/v" + str(api_version)

        # endpoints
        self.endpoint_full_text = self.host + self.api_version + '/full_text'
        self.endpoint_claim_only = self.host + self.api_version + '/claim_only'

        self.endpoint_data_feed = self.host + self.api_version + '/data'
        self.endpoint_patent_data = self.endpoint_data_feed + '/patent'

        self.endpoint_claim_parser = self.host + self.api_version + '/claim_parser'
        self.endpoint_ngram = self.endpoint_claim_parser + '/ngram'
        self.endpoint_keyphrase = self.endpoint_claim_parser + '/keyphrase'
        self.endpoint_claim_element = self.endpoint_claim_parser + '/claim_element'

    def check_service_status(self):
        """"Checks the status of all IP Street Endpoints"""
        pass

    def send(self, query_object):
        """Sends a instantiated query object, returns a list of dicts"""

        if isinstance(query_object, query.FullTextSearch):
            self.endpoint = self.endpoint_full_text
            self.payload = {"raw_text": query_object.raw_text, "q": query_object.q}
            pages = self.get_all_pages()
            assets = self.pages_to_assets(pages)
            return assets

        if isinstance(query_object, query.ClaimOnlySearch):
            self.endpoint = self.endpoint_claim_only
            self.payload = {"raw_text": query_object.raw_text, "q": query_object.q}
            pages = self.get_all_pages()
            assets = self.pages_to_assets(pages)
            return assets

        if isinstance(query_object, query.PatentData):
            self.endpoint = self.endpoint_patent_data
            self.payload = {'q': query_object.q}
            pages = self.get_all_pages()
            assets = self.pages_to_assets(pages)
            return assets

        if isinstance(query_object, query.NgramQuery):
            self.endpoint = self.endpoint_ngram
            self.payload = {'q': query_object.q}
            results = self.get_default_page()
            return results

        if isinstance(query_object, query.KeyPhraseQuery):
            self.endpoint = self.endpoint_keyphrase
            self.payload = {'q': query_object.q}
            results = self.get_default_page()
            return results

        if isinstance(query_object, query.ClaimElementsQuery):
            self.endpoint = self.endpoint_claim_element
            self.payload = {'q': query_object.q}
            results = self.get_default_page()
            return results

        else:
            raise SendError("The object you are attempting to send is not a valid query object")

    def get_default_page(self):
        """Get the first page of any given request, return results as json object"""
        r = requests.post(url=self.endpoint, headers=self.headers, data=json.dumps(self.payload))
        self.parse_response_codes(r, self.debug)
        results = r.json()

        if 'totalPage' in results:
            total_page_count = int(results['totalPage'])
            print('Downloading page 1 of {}'.format(str(total_page_count)))
        return results

    def get_all_pages(self):
        """Gets the 2nd page and all subsequent pages of a given response, returns all pages as json object"""
        pages = []

        first_page = self.get_default_page()
        pages.append(first_page)
        total_page_count = int(first_page['totalPage'])
        if total_page_count > 1:
            current_page_count = 2
            while current_page_count <= total_page_count:
                self.payload['q']['offset'] = current_page_count
                page = self.get_default_page()
                print('Downloading page {} of {}'.format(str(current_page_count),str(total_page_count)))
                pages.append(page)
                current_page_count += 1
        return pages

    def pages_to_assets(self,pages):
        """Converts all json object pages to a pythonic list of dicts"""
        assets = []
        # print(pages)
        for page in pages:
            assets.append(page['Assets'])

        assets = [j for i in assets for j in i]
        return assets

    def parse_response_codes(self, response, debug):
        """Parses the reponse code and raises a APIConnection error if not 200"""
        if response.status_code == 200:
            if debug:
                print("Request successful")
                return

        if response.status_code == 400:
            raise APIConnectionError("Invalid format or invalid data is specified in the request")

        if response.status_code == 401:
            raise APIConnectionError("Authentication credentials were missing or incorrect")

        if response.status_code == 403:
            raise APIConnectionError("The request was understood, but it has been refused")

        if response.status_code == 404:
            raise APIConnectionError("The URI requested is invalid or the requested resource does not exist")

        if response.status_code == 429:
            raise APIConnectionError("You are making requests too quickly, slow down and try again.")

        if response.status_code == 500:
            raise APIConnectionError("Something unexpected occurred.")

        if response.status_code == 502:
            raise APIConnectionError("IP Street service is down")

        if response.status_code == 503:
            raise APIConnectionError("IP Street service is up but overloaded with requests")<commit_msg>Added raise APIConnectionError for situations where status code is unknown.<commit_after>import json

import requests
from IPStreet.error import APIConnectionError, SendError

from IPStreet import query


class Client:
    """Basic IP Street API Client Object"""

    def __init__(self, apikey, api_version):

        self.debug = True

        # credentials
        self.apikey = apikey
        self.headers = {'x-api-key': self.apikey}

        # create most up-to-date endpoints
        self.host = 'https://api.ipstreet.com'
        self.api_version = "/v" + str(api_version)

        # endpoints
        self.endpoint_full_text = self.host + self.api_version + '/full_text'
        self.endpoint_claim_only = self.host + self.api_version + '/claim_only'

        self.endpoint_data_feed = self.host + self.api_version + '/data'
        self.endpoint_patent_data = self.endpoint_data_feed + '/patent'

        self.endpoint_claim_parser = self.host + self.api_version + '/claim_parser'
        self.endpoint_ngram = self.endpoint_claim_parser + '/ngram'
        self.endpoint_keyphrase = self.endpoint_claim_parser + '/keyphrase'
        self.endpoint_claim_element = self.endpoint_claim_parser + '/claim_element'

    def check_service_status(self):
        """"Checks the status of all IP Street Endpoints"""
        pass

    def send(self, query_object):
        """Sends a instantiated query object, returns a list of dicts"""

        if isinstance(query_object, query.FullTextSearch):
            self.endpoint = self.endpoint_full_text
            self.payload = {"raw_text": query_object.raw_text, "q": query_object.q}
            pages = self.get_all_pages()
            assets = self.pages_to_assets(pages)
            return assets

        if isinstance(query_object, query.ClaimOnlySearch):
            self.endpoint = self.endpoint_claim_only
            self.payload = {"raw_text": query_object.raw_text, "q": query_object.q}
            pages = self.get_all_pages()
            assets = self.pages_to_assets(pages)
            return assets

        if isinstance(query_object, query.PatentData):
            self.endpoint = self.endpoint_patent_data
            self.payload = {'q': query_object.q}
            pages = self.get_all_pages()
            assets = self.pages_to_assets(pages)
            return assets

        if isinstance(query_object, query.NgramQuery):
            self.endpoint = self.endpoint_ngram
            self.payload = {'q': query_object.q}
            results = self.get_default_page()
            return results

        if isinstance(query_object, query.KeyPhraseQuery):
            self.endpoint = self.endpoint_keyphrase
            self.payload = {'q': query_object.q}
            results = self.get_default_page()
            return results

        if isinstance(query_object, query.ClaimElementsQuery):
            self.endpoint = self.endpoint_claim_element
            self.payload = {'q': query_object.q}
            results = self.get_default_page()
            return results

        else:
            raise SendError("The object you are attempting to send is not a valid query object")

    def get_default_page(self):
        """Get the first page of any given request, return results as json object"""
        r = requests.post(url=self.endpoint, headers=self.headers, data=json.dumps(self.payload))
        self.parse_response_codes(r, self.debug)
        results = r.json()

        if 'totalPage' in results:
            total_page_count = int(results['totalPage'])
            print('Downloading page 1 of {}'.format(str(total_page_count)))
        return results

    def get_all_pages(self):
        """Gets the 2nd page and all subsequent pages of a given response, returns all pages as json object"""
        pages = []

        first_page = self.get_default_page()
        pages.append(first_page)
        total_page_count = int(first_page['totalPage'])
        if total_page_count > 1:
            current_page_count = 2
            while current_page_count <= total_page_count:
                self.payload['q']['offset'] = current_page_count
                page = self.get_default_page()
                print('Downloading page {} of {}'.format(str(current_page_count),str(total_page_count)))
                pages.append(page)
                current_page_count += 1
        return pages

    def pages_to_assets(self,pages):
        """Converts all json object pages to a pythonic list of dicts"""
        assets = []
        # print(pages)
        for page in pages:
            assets.append(page['Assets'])

        assets = [j for i in assets for j in i]
        return assets

    def parse_response_codes(self, response, debug):
        """Parses the reponse code and raises a APIConnection error if not 200"""
        if response.status_code == 200:
            if debug:
                print("Request successful")
                return

        if response.status_code == 400:
            raise APIConnectionError("Invalid format or invalid data is specified in the request")

        if response.status_code == 401:
            raise APIConnectionError("Authentication credentials were missing or incorrect")

        if response.status_code == 403:
            raise APIConnectionError("The request was understood, but it has been refused")

        if response.status_code == 404:
            raise APIConnectionError("The URI requested is invalid or the requested resource does not exist")

        if response.status_code == 429:
            raise APIConnectionError("You are making requests too quickly, slow down and try again.")

        if response.status_code == 500:
            raise APIConnectionError("Something unexpected occurred.")

        if response.status_code == 502:
            raise APIConnectionError("IP Street service is down")

        if response.status_code == 503:
            raise APIConnectionError("IP Street service is up but overloaded with requests")

        else:
            raise APIConnectionError("Response Code {}: An Unknown error has occured.".format(str(response.status_code)))
{'fraction_non_alphanumeric': 0.08822796377835009, 'fraction_numerical': 0.005815402508930797, 'mean_word_length': 2.4169741697416973, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7512763', 'n_tokens_mistral': 3273, 'n_tokens_neox': 3104, 'n_words': 873}