<commit_before>import json
import requests
import sys

from pylons import g


class AdzerkError(Exception): pass
class NotFound(AdzerkError): pass


def handle_response(response):
    if not (200 <= response.status_code <= 299):
        raise AdzerkError('response %s' % response.status_code)
    try:
        return json.loads(response.text)
    except ValueError:
        raise AdzerkError('bad response')


class Stub(object):
    def __init__(self, Id):
        self.Id = Id

    def _to_item(self):
        return {'Id': self.Id}


class Field(object):
    def __init__(self, name, optional=False):
        self.name = name
        self.optional = optional


class FieldSet(object):
    def __init__(self, *fields):
        self.fields = {field.name for field in fields}
        self.essentials = {field.name for field in fields if not field.optional}

    def to_set(self, exclude_optional=True):
        if exclude_optional:
            return self.essentials
        else:
            return self.fields

    def __iter__(self):
        for field_name in self.fields:
            yield field_name


class Base(object):
    _name = ''
    _base_url = 'http://api.adzerk.net/v1'
    _fields = FieldSet()

    @classmethod
    def _headers(cls):
        return {'X-Adzerk-ApiKey': g.secrets['az_selfserve_key'],
                'Content-Type': 'application/x-www-form-urlencoded'}

    def __init__(self, Id, _is_response=False, **attr):
        self.Id = Id
        missing = self._fields.to_set() - set(attr.keys())
        if missing:
            missing = ', '.join(missing)
            msg = 'missing required attributes: %s' % missing
            if _is_response:
                sys.stderr.write('WARNING: %s' % msg)
            else:
                raise ValueError(msg)

        for attr, val in attr.iteritems():
            self.__setattr__(attr, val, fail_on_unrecognized=(not _is_response))

    def __setattr__(self, attr, val, fail_on_unrecognized=True):
        if attr not in self._fields and attr != 'Id':
            msg = 'unrecognized attribute: %s' % attr
            if fail_on_unrecognized:
                raise ValueError(msg)
            else:
                pass
        object.__setattr__(self, attr, val)

    @classmethod
    def _from_item(cls, item):
        Id = item.pop('Id')
        thing = cls(Id, _is_response=True, **item)
        return thing

    def _to_item(self):
        item = {}
        if self.Id:
            item['Id'] = self.Id
        for attr in self._fields:
            if hasattr(self, attr):
                item[attr] = getattr(self, attr)
        return item

    def _to_data(self):
        return '%s=%s' % (self._name, json.dumps(self._to_item()))

    @classmethod
    def list(cls):
        url = '/'.join([cls._base_url, cls._name])
        response = requests.get(url, headers=cls._headers())
        content = handle_response(response)
        items = content.get('items')
        if items:
            return [cls._from_item(item) for item in items]
<commit_msg>Fix request encoding<commit_after>import json
import requests
import sys

from pylons import g


class AdzerkError(Exception): pass
class NotFound(AdzerkError): pass


def handle_response(response):
    if not (200 <= response.status_code <= 299):
        raise AdzerkError('response %s' % response.status_code)
    try:
        return json.loads(response.text)
    except ValueError:
        raise AdzerkError('bad response')


class Stub(object):
    def __init__(self, Id):
        self.Id = Id

    def _to_item(self):
        return {'Id': self.Id}


class Field(object):
    def __init__(self, name, optional=False):
        self.name = name
        self.optional = optional


class FieldSet(object):
    def __init__(self, *fields):
        self.fields = {field.name for field in fields}
        self.essentials = {field.name for field in fields if not field.optional}

    def to_set(self, exclude_optional=True):
        if exclude_optional:
            return self.essentials
        else:
            return self.fields

    def __iter__(self):
        for field_name in self.fields:
            yield field_name


class Base(object):
    _name = ''
    _base_url = 'http://api.adzerk.net/v1'
    _fields = FieldSet()

    @classmethod
    def _headers(cls):
        return {'X-Adzerk-ApiKey': g.secrets['az_selfserve_key'],
                'Content-Type': 'application/x-www-form-urlencoded'}

    def __init__(self, Id, _is_response=False, **attr):
        self.Id = Id
        missing = self._fields.to_set() - set(attr.keys())
        if missing:
            missing = ', '.join(missing)
            msg = 'missing required attributes: %s' % missing
            if _is_response:
                sys.stderr.write('WARNING: %s' % msg)
            else:
                raise ValueError(msg)

        for attr, val in attr.iteritems():
            self.__setattr__(attr, val, fail_on_unrecognized=(not _is_response))

    def __setattr__(self, attr, val, fail_on_unrecognized=True):
        if attr not in self._fields and attr != 'Id':
            msg = 'unrecognized attribute: %s' % attr
            if fail_on_unrecognized:
                raise ValueError(msg)
            else:
                pass
        object.__setattr__(self, attr, val)

    @classmethod
    def _from_item(cls, item):
        Id = item.pop('Id')
        thing = cls(Id, _is_response=True, **item)
        return thing

    def _to_item(self):
        item = {}
        if self.Id:
            item['Id'] = self.Id
        for attr in self._fields:
            if hasattr(self, attr):
                item[attr] = getattr(self, attr)
        return item

    def _to_data(self):
        return {self._name: json.dumps(self._to_item())}

    @classmethod
    def list(cls):
        url = '/'.join([cls._base_url, cls._name])
        response = requests.get(url, headers=cls._headers())
        content = handle_response(response)
        items = content.get('items')
        if items:
            return [cls._from_item(item) for item in items]

{'fraction_non_alphanumeric': 0.11098303968384654, 'fraction_numerical': 0.002305285690762391, 'mean_word_length': 2.181770560502881, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '331062', 'n_tokens_mistral': 1852, 'n_tokens_neox': 1795, 'n_words': 465}