<commit_before>from requests.auth import HTTPBasicAuth
from requests_oauthlib import OAuth1Session, OAuth2Session
from oauthlib.oauth2 import BackendApplicationClient
from oauthlib.common import to_unicode
from . import __version__

import requests
import json

TWITTER_API_VERSION = '1.1'
TWITTER_BASE_API_URL = 'https://%s.twitter.com'


class BirdyException(Exception):
    def __init__(self, msg, resource_url=None, request_method=None, status_code=None, error_code=None, headers=None):
        self._msg = msg
        self.request_method = request_method
        self.resource_url = resource_url
        self.status_code= status_code
        self.error_code = error_code
        self.headers = headers
        
    def __str__(self):
        if self.request_method and self.resource_url:
            return '%s (%s %s)' % (self._msg, self.request_method, self.resource_url)
        return self._msg
        

class TwitterClientError(BirdyException):
    pass


class TwitterApiError(BirdyException):
    def __init__(self, msg, response=None, request_method=None, error_code=None):
        kwargs = {}
        if response is not None:
            kwargs = {
                'status_code': response.status_code,
                'resource_url': response.url,
                'headers': response.headers,
            }

        super(TwitterApiError, self).__init__(
            msg,
            request_method=request_method,
            error_code=error_code,
            **kwargs
        )


class TwitterRateLimitError(TwitterApiError):
    pass


class TwitterAuthError(TwitterApiError):
    pass


class ApiComponent(object):
    def __init__(self, client, path=None):
        self._client = client
        self._path = path
    
    def __repr__(self):
        return '<ApiComponent: %s>' % self._path
    
    def __getitem__(self, path):
        if not self._path is None:
            path = '%s/%s' % (self._path, path)
        return ApiComponent(self._client, path)
    
    def __getattr__(self, path):
        return self[path]
    
    def get(self, **params):
        if self._path is None:
            raise TypeError('Calling get() on an empty API path is not supported.')
        return self._client.request('GET', self._path, **params)
        
    def post(self, **params):
        if self._path is None:
            raise TypeError('Calling post() on an empty API path is not supported.')
        return self._client.request('POST', self._path, **params)

    def get_path(self):
        return self._path


class BaseResponse(object):
    def __repr__(self):
        return '<%s: %s %s>' % (self.__class__.__name__, self.request_method, self.resource_url)


class ApiResponse(BaseResponse):
    def __init__(self, response, request_method, json_data):
        self.resource_url = response.url
        self.headers = response.headers
        self.request_method = request_method
        self.data = json_data


class StreamResponse(BaseResponse):
    def __init__(self, response, request_method, json_object_hook):
        self.resource_url = response.url
        self.headers = response.headers
        self.request_method = request_method
        self._stream_iter = response.iter_lines
        self._json_object_hook = json_object_hook
            
    def stream(self):
        for item in self._stream_iter():
            if item:
                try:
                    data = json.loads(item, object_hook=self._json_object_hook)
                except:
                    pass
                else:
                    yield data


class JSONObject(dict):
    def __getattr__(self, name):
        if name in self:
            return self[name]
        raise AttributeError('%s has no property named %s.' % (self.__class__.__name__, name))
    
    def __setattr__(self, *args):
        raise AttributeError('%s instances are read-only.' % self.__class__.__name__)
    __delattr__ = __setitem__ = __delitem__ = __setattr__
    
    def __repr__(self):
        return '<%s: %s>' % (self.__class__.__name__, dict.__repr__(self))


class BaseTwitterClient(object):<commit_msg>Decode item to string from utf-8 array<commit_after>from requests.auth import HTTPBasicAuth
from requests_oauthlib import OAuth1Session, OAuth2Session
from oauthlib.oauth2 import BackendApplicationClient
from oauthlib.common import to_unicode
from . import __version__

import requests
import json

TWITTER_API_VERSION = '1.1'
TWITTER_BASE_API_URL = 'https://%s.twitter.com'


class BirdyException(Exception):
    def __init__(self, msg, resource_url=None, request_method=None, status_code=None, error_code=None, headers=None):
        self._msg = msg
        self.request_method = request_method
        self.resource_url = resource_url
        self.status_code= status_code
        self.error_code = error_code
        self.headers = headers
        
    def __str__(self):
        if self.request_method and self.resource_url:
            return '%s (%s %s)' % (self._msg, self.request_method, self.resource_url)
        return self._msg
        

class TwitterClientError(BirdyException):
    pass


class TwitterApiError(BirdyException):
    def __init__(self, msg, response=None, request_method=None, error_code=None):
        kwargs = {}
        if response is not None:
            kwargs = {
                'status_code': response.status_code,
                'resource_url': response.url,
                'headers': response.headers,
            }

        super(TwitterApiError, self).__init__(
            msg,
            request_method=request_method,
            error_code=error_code,
            **kwargs
        )


class TwitterRateLimitError(TwitterApiError):
    pass


class TwitterAuthError(TwitterApiError):
    pass


class ApiComponent(object):
    def __init__(self, client, path=None):
        self._client = client
        self._path = path
    
    def __repr__(self):
        return '<ApiComponent: %s>' % self._path
    
    def __getitem__(self, path):
        if not self._path is None:
            path = '%s/%s' % (self._path, path)
        return ApiComponent(self._client, path)
    
    def __getattr__(self, path):
        return self[path]
    
    def get(self, **params):
        if self._path is None:
            raise TypeError('Calling get() on an empty API path is not supported.')
        return self._client.request('GET', self._path, **params)
        
    def post(self, **params):
        if self._path is None:
            raise TypeError('Calling post() on an empty API path is not supported.')
        return self._client.request('POST', self._path, **params)

    def get_path(self):
        return self._path


class BaseResponse(object):
    def __repr__(self):
        return '<%s: %s %s>' % (self.__class__.__name__, self.request_method, self.resource_url)


class ApiResponse(BaseResponse):
    def __init__(self, response, request_method, json_data):
        self.resource_url = response.url
        self.headers = response.headers
        self.request_method = request_method
        self.data = json_data


class StreamResponse(BaseResponse):
    def __init__(self, response, request_method, json_object_hook):
        self.resource_url = response.url
        self.headers = response.headers
        self.request_method = request_method
        self._stream_iter = response.iter_lines
        self._json_object_hook = json_object_hook
            
    def stream(self):
        for item in self._stream_iter():
            if item:
                try:
                    data = json.loads(item.decode('utf-8'), object_hook=self._json_object_hook)
                except:
                    pass
                else:
                    yield data


class JSONObject(dict):
    def __getattr__(self, name):
        if name in self:
            return self[name]
        raise AttributeError('%s has no property named %s.' % (self.__class__.__name__, name))
    
    def __setattr__(self, *args):
        raise AttributeError('%s instances are read-only.' % self.__class__.__name__)
    __delattr__ = __setitem__ = __delitem__ = __setattr__
    
    def __repr__(self):
        return '<%s: %s>' % (self.__class__.__name__, dict.__repr__(self))


class BaseTwitterClient(object):
{'fraction_non_alphanumeric': 0.09317823821640615, 'fraction_numerical': 0.0014540167211922936, 'mean_word_length': 2.4827004219409283, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4844923', 'n_tokens_mistral': 2377, 'n_tokens_neox': 2330, 'n_words': 579}