<commit_before>import json

from six.moves.urllib_parse import urlparse, parse_qsl
import os
import pytest
import six

if six.PY2:
    try:
        from six import cStringIO as BufferIO
    except ImportError:
        from six import StringIO as BufferIO
else:
    from io import BytesIO as BufferIO

TESTS_PATH = os.path.abspath(os.path.dirname(__file__) + os.path.sep + '..')


def read(path, mode='rb'):
    if not os.path.isabs(path):
        path = os.path.join(TESTS_PATH, path)

    path = os.path.abspath(path)

    with open(path, mode) as fp:
        return BufferIO(fp.read())


def authenticated_response(path=None, data=None):
    def callback(request):
        h = request.headers

        if h.get('trakt-user-login') == 'mock' and h.get('trakt-user-token') == 'mock':
            return 200, {}, read(path) if path else data

        if h.get('Authorization') == 'Bearer mock':
            return 200, {}, read(path) if path else data

        return 401, {}, ''

    return callback


def pagination_response(path=None, authenticated=False):
    # Read file
    data = read(path)

    if not data:
        raise Exception('Invalid path specified: %r' % path)

    # Parse file
    collection = json.load(data)

    # Construct request callback
    def callback(request):
        if authenticated and not is_authenticated(request.headers):
            return 401, {}, ''

        # Parse url
        url = urlparse(request.url)
        query = dict(parse_qsl(url.query))

        # Retrieve parameters
        page = int(query.get('page', 1))
        limit = int(query.get('limit', 10))

        # Retrieve page items
        start = (page - 1) * limit
        end = page * limit

        items = collection[start:end]

        if not items:
            return 404, {}, ''

        # Return page response
        return (
            200, {
                'Content-Type':             'application/json',
                'X-Pagination-Page':        str(page),
                'X-Pagination-Limit':       str(limit),
                'X-Pagination-Page-Count':  str(len(collection) / limit),
                'X-Pagination-Item-Count':  str(len(collection))<commit_msg>Fixed Python 3+ compatibility issue in tests<commit_after>import json

from six.moves.urllib_parse import urlparse, parse_qsl
import os
import pytest
import six

if six.PY2:
    try:
        from six import cStringIO as BufferIO
    except ImportError:
        from six import StringIO as BufferIO
else:
    from io import BytesIO as BufferIO

TESTS_PATH = os.path.abspath(os.path.dirname(__file__) + os.path.sep + '..')


def read(path, mode='rb'):
    if not os.path.isabs(path):
        path = os.path.join(TESTS_PATH, path)

    path = os.path.abspath(path)

    with open(path, mode) as fp:
        return BufferIO(fp.read())


def authenticated_response(path=None, data=None):
    def callback(request):
        h = request.headers

        if h.get('trakt-user-login') == 'mock' and h.get('trakt-user-token') == 'mock':
            return 200, {}, read(path) if path else data

        if h.get('Authorization') == 'Bearer mock':
            return 200, {}, read(path) if path else data

        return 401, {}, ''

    return callback


def pagination_response(path=None, authenticated=False):
    if not os.path.isabs(path):
        path = os.path.join(TESTS_PATH, path)

    path = os.path.abspath(path)

    # Parse file
    with open(path, 'r') as fp:
        collection = json.load(fp)

    # Construct request callback
    def callback(request):
        if authenticated and not is_authenticated(request.headers):
            return 401, {}, ''

        # Parse url
        url = urlparse(request.url)
        query = dict(parse_qsl(url.query))

        # Retrieve parameters
        page = int(query.get('page', 1))
        limit = int(query.get('limit', 10))

        # Retrieve page items
        start = (page - 1) * limit
        end = page * limit

        items = collection[start:end]

        if not items:
            return 404, {}, ''

        # Return page response
        return (
            200, {
                'Content-Type':             'application/json',
                'X-Pagination-Page':        str(page),
                'X-Pagination-Limit':       str(limit),
                'X-Pagination-Page-Count':  str(len(collection) / limit),
                'X-Pagination-Item-Count':  str(len(collection))
{'fraction_non_alphanumeric': 0.11912296564195299, 'fraction_numerical': 0.010623869801084991, 'mean_word_length': 2.1562054208273893, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '772760', 'n_tokens_mistral': 1402, 'n_tokens_neox': 1314, 'n_words': 349}