<commit_before>
from tornado.gen import coroutine, Return
from tornado.web import HTTPError
from common.options import options

import tornado.httpclient

import discover
import singleton
import ipaddr
import rabbitrpc
import logging
import socket
import urllib
import ujson
import jsonrpc

from . import ElapsedTime


class Internal(rabbitrpc.RabbitMQJsonRPC):
    """
    Internal class is user for 'internal' communication between services across the environment.
    """
    __metaclass__ = singleton.Singleton

    def __init__(self):
        self.client = tornado.httpclient.AsyncHTTPClient()

        self.internal_locations = map(ipaddr.IPNetwork, options.internal_restrict)
        self.broker = options.internal_broker

        super(Internal, self).__init__()

    @coroutine
    def get(self, service, url, data, use_json=True, discover_service=True, timeout=20, network="internal"):
        """
        Requests a http GET page.

        :param service: Service ID the page is requested from
        :param url: Last par of the request url
        :param data: a disc to be converted to request arguments
        :param use_json: whenever should the result be converted to json or not
        :param discover_service: if True, <service> argument is a service ID,
            if False, <service> is just server location.
        :param timeout: a request timeout in seconds
        :param network: a network to make request in. Default is internal network
        :return: Requested data
        """
        if discover_service:
            try:
                service_location = yield discover.cache.get_service(service, network=network)
            except discover.DiscoveryError as e:
                raise InternalError(e.code, "Failed to discover '{0}': ".format(service) + e.message)
        else:
            service_location = service

        if service_location is None:
            raise InternalError(404, "Requesting empty url")

        timer = ElapsedTime("get -> {0}@{1}".format(url, service))

        try:
            request = tornado.httpclient.HTTPRequest(
                service_location + "/" + url + "?" + urllib.urlencode(data),
                method='GET',
                request_timeout=timeout
            )

            result = yield self.client.fetch(request)

        except tornado.httpclient.HTTPError as e:
            raise InternalError(e.code, e.response.body if hasattr(e.response, "body") else "", e.response)

        except socket.error as e:
            logging.exception("get {0}: {1}".format(service, url))
            raise InternalError(599, "Connection error: " + e.message, None)

        finally:
            logging.info(timer.done())

        raise Return(Internal.__parse_result__(result, use_json=use_json))

    def is_internal(self, remote_ip):
        """
        Checks if the IP is considered internal (is inside the internal environment).
        Use 'restrict_internal' command line argument to add IP.
        """
        return any((ipaddr.IPAddress(remote_ip) in network) for network in self.internal_locations)

    @coroutine
    def listen(self, service_name, on_receive):
        yield super(Internal, self).listen(self.broker, service_name, on_receive)

    @staticmethod
    def __parse_result__(result, use_json=True, return_headers=False):
        data = result.body

        if not use_json:
            return data

        if len(data) == 0:
            return None

        try:
            content = ujson.loads(data)
        except (KeyError, ValueError):
            raise IndexError(400, "Body is corrupted: " + data)

        if return_headers:
            return content, result.headers

        return content

    @coroutine
    def post(self, service, url, data, use_json=True, discover_service=True,
             timeout=20, network="internal", return_headers=False):
        """
        Posts a http request to a certain service

        :param service: Service ID the page is requested from
        :param url: Last par of the request url
        :param data: a disc to be converted to request arguments
        :param use_json: whenever should the result be converted to json or not
        :param discover_service: if True, <service> argument is a service ID,
            if False, <service> is just server location.
        :param timeout: a request timeout in seconds
        :param network: a network to make request in. Default is internal network
        :param return_headers: if True, a tuple (result, headers) instead of just result will be returned
        :return: Requested data
        """
        if discover_service:
            try:
                service_location = yield discover.cache.get_service(service, network=network)
            except discover.DiscoveryError as e:
                raise InternalError(e.code, "Failed to discover '{0}': " + e.message)
        else:
            service_location = service

        if service_location is None:
            raise InternalError(404, "Requesting empty url")

        timer = ElapsedTime("post -> {0}@{1}".format(url, service))

        try:
            request = tornado.httpclient.HTTPRequest(
                service_location + "/" + url,
                method='POST',
                body=urllib.urlencode(data),
                request_timeout=timeout)

            result = yield self.client.fetch(request)

        except tornado.httpclient.HTTPError as e:
            raise InternalError(e.code, e.response.body if hasattr(e.response, "body") else "", e.response)

        except socket.error as e:
            raise InternalError(599, "Connection error: " + e.message, None)
        finally:
            logging.info(timer.done())

        raise Return(Internal.__parse_result__(result, use_json=use_json, return_headers=return_headers))

    @coroutine
    def request(self, service, method, timeout=jsonrpc.JSONRPC_TIMEOUT, *args, **kwargs):
        """
        Makes a RabbitMQ RPC request to a certain service.

        :param service: Service ID the page is requested from
        :param method: Service Method to call (as described in internal handler)
        :param args, kwargs: Arguments to send to the method
        :param timeout: A timeout
        
        :returns Request response from service from the other side
        :raises InternalError on either connection issues or the requested service responded so
        
        """<commit_msg>Added X-Api-Version for internal requests<commit_after>
from tornado.gen import coroutine, Return
from tornado.web import HTTPError
from common.options import options

import tornado.httpclient

import discover
import singleton
import ipaddr
import rabbitrpc
import logging
import socket
import urllib
import ujson
import jsonrpc

from . import ElapsedTime


class Internal(rabbitrpc.RabbitMQJsonRPC):
    """
    Internal class is user for 'internal' communication between services across the environment.
    """
    __metaclass__ = singleton.Singleton

    def __init__(self):
        self.client = tornado.httpclient.AsyncHTTPClient()

        self.internal_locations = map(ipaddr.IPNetwork, options.internal_restrict)
        self.broker = options.internal_broker

        super(Internal, self).__init__()

    @coroutine
    def get(self, service, url, data, use_json=True, discover_service=True, timeout=20, network="internal"):
        """
        Requests a http GET page.

        :param service: Service ID the page is requested from
        :param url: Last par of the request url
        :param data: a disc to be converted to request arguments
        :param use_json: whenever should the result be converted to json or not
        :param discover_service: if True, <service> argument is a service ID,
            if False, <service> is just server location.
        :param timeout: a request timeout in seconds
        :param network: a network to make request in. Default is internal network
        :return: Requested data
        """
        if discover_service:
            try:
                service_location = yield discover.cache.get_service(service, network=network)
            except discover.DiscoveryError as e:
                raise InternalError(e.code, "Failed to discover '{0}': ".format(service) + e.message)
        else:
            service_location = service

        if service_location is None:
            raise InternalError(404, "Requesting empty url")

        timer = ElapsedTime("get -> {0}@{1}".format(url, service))

        try:
            request = tornado.httpclient.HTTPRequest(
                service_location + "/" + url + "?" + urllib.urlencode(data),
                method='GET',
                request_timeout=timeout,
                headers={
                    "X-Api-Version": options.api_version
                }
            )

            result = yield self.client.fetch(request)

        except tornado.httpclient.HTTPError as e:
            raise InternalError(e.code, e.response.body if hasattr(e.response, "body") else "", e.response)

        except socket.error as e:
            logging.exception("get {0}: {1}".format(service, url))
            raise InternalError(599, "Connection error: " + e.message, None)

        finally:
            logging.info(timer.done())

        raise Return(Internal.__parse_result__(result, use_json=use_json))

    def is_internal(self, remote_ip):
        """
        Checks if the IP is considered internal (is inside the internal environment).
        Use 'restrict_internal' command line argument to add IP.
        """
        return any((ipaddr.IPAddress(remote_ip) in network) for network in self.internal_locations)

    @coroutine
    def listen(self, service_name, on_receive):
        yield super(Internal, self).listen(self.broker, service_name, on_receive)

    @staticmethod
    def __parse_result__(result, use_json=True, return_headers=False):
        data = result.body

        if not use_json:
            return data

        if len(data) == 0:
            return None

        try:
            content = ujson.loads(data)
        except (KeyError, ValueError):
            raise IndexError(400, "Body is corrupted: " + data)

        if return_headers:
            return content, result.headers

        return content

    @coroutine
    def post(self, service, url, data, use_json=True, discover_service=True,
             timeout=20, network="internal", return_headers=False):
        """
        Posts a http request to a certain service

        :param service: Service ID the page is requested from
        :param url: Last par of the request url
        :param data: a disc to be converted to request arguments
        :param use_json: whenever should the result be converted to json or not
        :param discover_service: if True, <service> argument is a service ID,
            if False, <service> is just server location.
        :param timeout: a request timeout in seconds
        :param network: a network to make request in. Default is internal network
        :param return_headers: if True, a tuple (result, headers) instead of just result will be returned
        :return: Requested data
        """
        if discover_service:
            try:
                service_location = yield discover.cache.get_service(service, network=network)
            except discover.DiscoveryError as e:
                raise InternalError(e.code, "Failed to discover '{0}': " + e.message)
        else:
            service_location = service

        if service_location is None:
            raise InternalError(404, "Requesting empty url")

        timer = ElapsedTime("post -> {0}@{1}".format(url, service))

        try:
            request = tornado.httpclient.HTTPRequest(
                service_location + "/" + url,
                method='POST',
                body=urllib.urlencode(data),
                request_timeout=timeout,
                headers={
                    "X-Api-Version": options.api_version
                })

            result = yield self.client.fetch(request)

        except tornado.httpclient.HTTPError as e:
            raise InternalError(e.code, e.response.body if hasattr(e.response, "body") else "", e.response)

        except socket.error as e:
            raise InternalError(599, "Connection error: " + e.message, None)
        finally:
            logging.info(timer.done())

        raise Return(Internal.__parse_result__(result, use_json=use_json, return_headers=return_headers))

    @coroutine
    def request(self, service, method, timeout=jsonrpc.JSONRPC_TIMEOUT, *args, **kwargs):
        """
        Makes a RabbitMQ RPC request to a certain service.

        :param service: Service ID the page is requested from
        :param method: Service Method to call (as described in internal handler)
        :param args, kwargs: Arguments to send to the method
        :param timeout: A timeout
        
        :returns Request response from service from the other side
        :raises InternalError on either connection issues or the requested service responded so
        
        """
{'fraction_non_alphanumeric': 0.08125, 'fraction_numerical': 0.00426829268292683, 'mean_word_length': 2.3930695629687095, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2061927', 'n_tokens_mistral': 3434, 'n_tokens_neox': 3303, 'n_words': 1144}