<commit_before>import boto
import functools
import json
import logging
import random
import requests
import urllib
from urlparse import urlparse

from boto.connection import AWSAuthConnection
from boto.s3.connection import S3Connection

log = logging.getLogger(__name__)

class Endpoint(object):
    def __init__(self, host, port, secure,
                 access_key=None, secret_key=None, region=None, zone=None):
        self.host = host
        default_port = 443 if secure else 80
        self.port = port or default_port
        self.secure = secure
        self.access_key = access_key
        self.secret_key = secret_key
        self.region = region
        self.zone = zone

    def __eq__(self, other):
        if self.host != other.host:
            return False
        if self.port == other.port:
            return True
        # if self and other are mixed http/https with default ports,
        # i.e. http://example.com and https://example.com, consider
        # them the same
        def diff_only_default_ports(a, b):
            return a.secure and a.port == 443 and not b.secure and b.port == 80
        return (diff_only_default_ports(self, other) or
                diff_only_default_ports(other, self))

    def __repr__(self):
        return 'Endpoint(host={host}, port={port}, secure={secure})'.format(
            host=self.host,
            port=self.port,
            secure=self.secure)

    def __str__(self):
        scheme = 'https' if self.secure else 'http'
        return '{scheme}://{host}:{port}'.format(scheme=scheme,
                                                 host=self.host,
                                                 port=self.port)

class ClientException(Exception):
    pass
class InvalidProtocol(ClientException):
    pass
class InvalidHost(ClientException):
    pass
class InvalidZone(ClientException):
    pass
class ZoneNotFound(ClientException):
    pass

def parse_endpoint(endpoint):
    url = urlparse(endpoint)
    if url.scheme not in ['http', 'https']:
        raise InvalidProtocol('invalid protocol %r' % url.scheme)
    if not url.hostname:
        raise InvalidHost('no hostname in %r' % endpoint)
    return Endpoint(url.hostname, url.port, url.scheme == 'https')

class HttpError(ClientException):
    def __init__(self, code, body):
        self.code = code
        self.str_code = str(code)
        self.body = body
        self.message = 'Http error code %s content %s' % (code, body)
    def __str__(self):
        return self.message
class NotFound(HttpError):
    pass
code_to_exc = {
    404: NotFound,
    }

def boto_call(func):
    @functools.wraps(func)
    def translate_exception(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except boto.exception.S3ResponseError as e:
            raise code_to_exc.get(e.status, HttpError)(e.status, e.body)
    return translate_exception


"""
Adapted from the build_request() method of boto.connection
"""

def _build_request(conn, method, basepath='', resource = '', headers=None,
                   data=None, special_first_param=None, params=None):
    path = conn.calling_format.build_path_base(basepath, resource)
    auth_path = conn.calling_format.build_auth_path(basepath, resource)
    host = conn.calling_format.build_host(conn.server_name(), '')

    if special_first_param:
        path += '?' + special_first_param
        boto.log.debug('path=%s' % path)
        auth_path += '?' + special_first_param
        boto.log.debug('auth_path=%s' % auth_path)

    return AWSAuthConnection.build_base_http_request(
        conn, method, path, auth_path, params, headers, data, host)

def check_result_status(result):
    if result.status_code / 100 != 2:
        raise code_to_exc.get(result.status_code,
                              HttpError)(result.status_code, result.content)
def url_safe(component):
    if isinstance(component, basestring):
        string = component.encode('utf8')
    else:
        string = str(component)
    return urllib.quote(string)

def request(connection, type_, resource, params=None, headers=None,
            data=None, expect_json=True, special_first_param=None):
    if headers is None:
        headers = {}

    if type_ in ['put', 'post']:
        headers['Content-Type'] = 'application/json; charset=UTF-8'

    request_data = data if data else ''
    if params is None:
        params = {}
    safe_params = dict([(k, url_safe(v)) for k, v in params.iteritems()])
    request = _build_request(connection,
                             type_.upper(),
                             resource=resource,
                             special_first_param=special_first_param,
                             headers=headers,
                             data=request_data,
                             params=safe_params)

    url = '{protocol}://{host}{path}'.format(protocol=request.protocol,
                                             host=request.host,
                                             path=request.path)

    request.authorize(connection=connection)

    handler = getattr(requests, type_)
    boto.log.debug('url = %r\nparams=%r\nheaders=%r\ndata=%r',
                   url, params, request.headers, data)
    result = handler(url, params=params, headers=request.headers, data=data)

    check_result_status(result)

    if data or not expect_json:
        return result.raw
    return result.json()

def get_metadata(connection, section, name):
    return request(connection, 'get', 'admin/metadata/' + section,
                   params=dict(key=name))

def update_metadata(connection, section, name, metadata):
    if not isinstance(metadata, basestring):
        metadata = json.dumps(metadata)
    return request(connection, 'put', 'admin/metadata/' + section,
                   params=dict(key=name), data=metadata)

def delete_metadata(connection, section, name):
    return request(connection, 'delete', 'admin/metadata/' + section,
                   params=dict(key=name), expect_json=False)

def get_metadata_sections(connection):
    return request(connection, 'get', 'admin/metadata')

def list_metadata_keys(connection, section):<commit_msg>remove requests, use boto directly<commit_after>import boto
import functools
import json
import logging
import random
import urllib
from urlparse import urlparse

from boto.connection import AWSAuthConnection
from boto.exception import BotoServerError
from boto.s3.connection import S3Connection

log = logging.getLogger(__name__)

class Endpoint(object):
    def __init__(self, host, port, secure,
                 access_key=None, secret_key=None, region=None, zone=None):
        self.host = host
        default_port = 443 if secure else 80
        self.port = port or default_port
        self.secure = secure
        self.access_key = access_key
        self.secret_key = secret_key
        self.region = region
        self.zone = zone

    def __eq__(self, other):
        if self.host != other.host:
            return False
        if self.port == other.port:
            return True
        # if self and other are mixed http/https with default ports,
        # i.e. http://example.com and https://example.com, consider
        # them the same
        def diff_only_default_ports(a, b):
            return a.secure and a.port == 443 and not b.secure and b.port == 80
        return (diff_only_default_ports(self, other) or
                diff_only_default_ports(other, self))

    def __repr__(self):
        return 'Endpoint(host={host}, port={port}, secure={secure})'.format(
            host=self.host,
            port=self.port,
            secure=self.secure)

    def __str__(self):
        scheme = 'https' if self.secure else 'http'
        return '{scheme}://{host}:{port}'.format(scheme=scheme,
                                                 host=self.host,
                                                 port=self.port)

class ClientException(Exception):
    pass
class InvalidProtocol(ClientException):
    pass
class InvalidHost(ClientException):
    pass
class InvalidZone(ClientException):
    pass
class ZoneNotFound(ClientException):
    pass

def parse_endpoint(endpoint):
    url = urlparse(endpoint)
    if url.scheme not in ['http', 'https']:
        raise InvalidProtocol('invalid protocol %r' % url.scheme)
    if not url.hostname:
        raise InvalidHost('no hostname in %r' % endpoint)
    return Endpoint(url.hostname, url.port, url.scheme == 'https')

class HttpError(ClientException):
    def __init__(self, code, body):
        self.code = code
        self.str_code = str(code)
        self.body = body
        self.message = 'Http error code %s content %s' % (code, body)
    def __str__(self):
        return self.message
class NotFound(HttpError):
    pass
code_to_exc = {
    404: NotFound,
    }

def boto_call(func):
    @functools.wraps(func)
    def translate_exception(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except boto.exception.S3ResponseError as e:
            raise code_to_exc.get(e.status, HttpError)(e.status, e.body)
    return translate_exception


"""
Adapted from the build_request() method of boto.connection
"""

def _build_request(conn, method, basepath='', resource = '', headers=None,
                   data=None, special_first_param=None, params=None):

    path = conn.calling_format.build_path_base(basepath, resource)
    auth_path = conn.calling_format.build_auth_path(basepath, resource)
    host = conn.calling_format.build_host(conn.server_name(), '')

    if special_first_param:
        path += '?' + special_first_param
        boto.log.debug('path=%s' % path)
        auth_path += '?' + special_first_param
        boto.log.debug('auth_path=%s' % auth_path)

    return AWSAuthConnection.build_base_http_request(
        conn, method, path, auth_path, params, headers, data, host)

def check_result_status(result):
    if result.status / 100 != 2:
        raise code_to_exc.get(result.status,
                              HttpError)(result.status, result.reason)
def url_safe(component):
    if isinstance(component, basestring):
        string = component.encode('utf8')
    else:
        string = str(component)
    return urllib.quote(string)

def request(connection, type_, resource, params=None, headers=None,
            data=None, expect_json=True, special_first_param=None, _retries=3):
    if headers is None:
        headers = {}

    if type_ in ['put', 'post']:
        headers['Content-Type'] = 'application/json; charset=UTF-8'

    request_data = data if data else ''
    if params is None:
        params = {}
    safe_params = dict([(k, url_safe(v)) for k, v in params.iteritems()])
    request = _build_request(connection,
                             type_.upper(),
                             resource=resource,
                             special_first_param=special_first_param,
                             headers=headers,
                             data=request_data,
                             params=safe_params)

    url = '{protocol}://{host}{path}'.format(protocol=request.protocol,
                                             host=request.host,
                                             path=request.path)

    request.authorize(connection=connection)

    boto.log.debug('url = %r\nparams=%r\nheaders=%r\ndata=%r',
                   url, params, request.headers, data)
    try:
        result = connection.make_request(
            request.method,
            request.path,
            request.body,
            headers=request.headers,
            override_num_retries=_retries)
    except BotoServerError as error:
        check_result_status(error)

    check_result_status(result)

    if data or not expect_json:
        return result.raw

    return json.loads(result.read())


def get_metadata(connection, section, name):
    return request(connection, 'get', 'admin/metadata/' + section,
                   params=dict(key=name))

def update_metadata(connection, section, name, metadata):
    if not isinstance(metadata, basestring):
        metadata = json.dumps(metadata)
    return request(connection, 'put', 'admin/metadata/' + section,
                   params=dict(key=name), data=metadata)

def delete_metadata(connection, section, name):
    return request(connection, 'delete', 'admin/metadata/' + section,
                   params=dict(key=name), expect_json=False)

def get_metadata_sections(connection):
    return request(connection, 'get', 'admin/metadata')

def list_metadata_keys(connection, section):
{'fraction_non_alphanumeric': 0.1042793509385937, 'fraction_numerical': 0.0035793827553293033, 'mean_word_length': 2.470328457079768, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4002970', 'n_tokens_mistral': 3553, 'n_tokens_neox': 3481, 'n_words': 849}