<commit_before>import pytest

import cocrawler.robots as robots
from cocrawler.robots import is_plausible_robots


def test_preprocess_robots():
    robots_txt = '''
foo
#bar

baz
'''
    ret = '''foo\nbaz\n'''
    assert robots.preprocess_robots(robots_txt, 'foo', {}) == (ret, False)

    robots_txt = 'User-AgEnT: cOcRaWlEr\nAllow: /'
    json_log = {}
    ret = 'User-AgEnT: cOcRaWlEr\nAllow: /\n'
    assert robots.preprocess_robots(robots_txt, 'CoCrAwLeR', json_log) == (ret, True)
    assert json_log == {'action-lines': 1, 'mentions-us': True, 'size': 30, 'user-agents': 1}

    assert robots.preprocess_robots('', 'foo', {}) == ('', False)
    assert robots.preprocess_robots('foo', 'foo', {}) == ('foo\n', False)


def test_strip_bom():
    robots_txt = b'\xef\xbb\xbf'
    assert robots.strip_bom(robots_txt) == b''
    robots_txt = b'\xef\xbb\xbf  '
    assert robots.strip_bom(robots_txt) == b'  '

    robots_txt = b'\xfe\xff'
    assert robots.strip_bom(robots_txt) == b''
    robots_txt = b'\xfe\xfffoo'
    assert robots.strip_bom(robots_txt) == b'foo'

    robots_txt = b'\xff\xfe'
    assert robots.strip_bom(robots_txt) == b''


def test_robots():
    '''
    There's already end-to-end testing for the normal functionality.
    Exercise only the weird stuff here.
    '''
    robots_txt = b'<'
    plausible, message = is_plausible_robots(robots_txt)
    assert not plausible
    assert len(message)

    robots_txt = b''  # application/x-empty
    plausible, message = is_plausible_robots(robots_txt)
    assert plausible
    assert not len(message)

    robots_txt = b'x'*1000001
    plausible, message = is_plausible_robots(robots_txt)
    assert not plausible
    assert len(message)

    robots_txt = b'foo'
    plausible, message = is_plausible_robots(robots_txt)
    assert plausible
    assert not len(message)
<commit_msg>naming<commit_after>import pytest

import cocrawler.robots as robots
from cocrawler.robots import is_plausible_robots


def test_preprocess_robots():
    robots_txt = '''
foo
#bar

baz
'''
    ret = '''foo\nbaz\n'''
    assert robots.preprocess_robots(robots_txt, 'foo', {}) == (ret, False)

    robots_txt = 'User-AgEnT: cOcRaWlEr\nAllow: /'
    json_log = {}
    ret = 'User-AgEnT: cOcRaWlEr\nAllow: /\n'
    assert robots.preprocess_robots(robots_txt, 'CoCrAwLeR', json_log) == (ret, True)
    assert json_log == {'action-lines': 1, 'mentions-us': True, 'size': 30, 'user-agents': 1}

    assert robots.preprocess_robots('', 'foo', {}) == ('', False)
    assert robots.preprocess_robots('foo', 'foo', {}) == ('foo\n', False)


def test_strip_bom():
    robots_txt = b'\xef\xbb\xbf'
    assert robots.strip_bom(robots_txt) == b''
    robots_txt = b'\xef\xbb\xbf  '
    assert robots.strip_bom(robots_txt) == b'  '

    robots_txt = b'\xfe\xff'
    assert robots.strip_bom(robots_txt) == b''
    robots_txt = b'\xfe\xfffoo'
    assert robots.strip_bom(robots_txt) == b'foo'

    robots_txt = b'\xff\xfe'
    assert robots.strip_bom(robots_txt) == b''


def test_is_plausible_robots():
    robots_txt = b'<'
    plausible, message = is_plausible_robots(robots_txt)
    assert not plausible
    assert len(message)

    robots_txt = b''  # application/x-empty
    plausible, message = is_plausible_robots(robots_txt)
    assert plausible
    assert not len(message)

    robots_txt = b'x'*1000001
    plausible, message = is_plausible_robots(robots_txt)
    assert not plausible
    assert len(message)

    robots_txt = b'foo'
    plausible, message = is_plausible_robots(robots_txt)
    assert plausible
    assert not len(message)

{'fraction_non_alphanumeric': 0.13640167364016736, 'fraction_numerical': 0.006136680613668061, 'mean_word_length': 3.6753585397653192, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4089669', 'n_tokens_mistral': 1435, 'n_tokens_neox': 1363, 'n_words': 266}