# Path: cocrawler/robots.py
# def is_plausible_robots(body_bytes):
#     '''
#     Did you know that some sites have a robots.txt that's a 100 megabyte video file?
# 
#     file magic mimetype is 'text' or similar -- too expensive, 3ms per call
#     '''
#     if body_bytes.startswith(b'<'):
#         return False, 'robots appears to be html or xml'
#     if len(body_bytes) > 1000000:
#         return False, 'robots is too big'
# 
#     return True, ''
# 
# def robots_facets(text, robotname, json_log):
#     user_agents = re.findall(r'^ \s* User-Agent: \s* (.*) \s* (?:\#.*)?', text, re.X | re.I | re.M)
#     action_lines = len(re.findall(r'^ \s* (allow|disallow|crawl-delay):', text, re.X | re.I | re.M))
# 
#     user_agents = list(set([u.lower() for u in user_agents]))
# 
#     mentions_us = robotname.lower() in user_agents
# 
#     if mentions_us:
#         json_log['mentions_us'] = True
#     if user_agents:
#         json_log['user_agents'] = len(user_agents)
#     if action_lines:
#         json_log['action_lines'] = action_lines
#     if text:
#         json_log['size'] = len(text)


import pytest
import cocrawler.robots as robots
from cocrawler.robots import is_plausible_robots, robots_facets
    assert robots.strip_bom(robots_txt) == b''
    robots_txt = b'\xef\xbb\xbf  '
    assert robots.strip_bom(robots_txt) == b'  '

    robots_txt = b'\xfe\xff'
    assert robots.strip_bom(robots_txt) == b''
    robots_txt = b'\xfe\xfffoo'
    assert robots.strip_bom(robots_txt) == b'foo'

    robots_txt = b'\xff\xfe'
    assert robots.strip_bom(robots_txt) == b''


def test_is_plausible_robots():
    robots_txt = b'<'
    plausible, message = is_plausible_robots(robots_txt)
    assert not plausible
    assert len(message)

    robots_txt = b''  # application/x-empty
    plausible, message = is_plausible_robots(robots_txt)
    assert plausible
    assert not len(message)

    robots_txt = b'x'*1000001
    plausible, message = is_plausible_robots(robots_txt)
    assert not plausible
    assert len(message)

    robots_txt = b'foo'

# Path: cocrawler/robots.py
# def is_plausible_robots(body_bytes):
#     '''
#     Did you know that some sites have a robots.txt that's a 100 megabyte video file?
# 
#     file magic mimetype is 'text' or similar -- too expensive, 3ms per call
#     '''
#     if body_bytes.startswith(b'<'):
#         return False, 'robots appears to be html or xml'
#     if len(body_bytes) > 1000000:
#         return False, 'robots is too big'
# 
#     return True, ''
# 
# def robots_facets(text, robotname, json_log):
#     user_agents = re.findall(r'^ \s* User-Agent: \s* (.*) \s* (?:\#.*)?', text, re.X | re.I | re.M)
#     action_lines = len(re.findall(r'^ \s* (allow|disallow|crawl-delay):', text, re.X | re.I | re.M))
# 
#     user_agents = list(set([u.lower() for u in user_agents]))
# 
#     mentions_us = robotname.lower() in user_agents
# 
#     if mentions_us:
#         json_log['mentions_us'] = True
#     if user_agents:
#         json_log['user_agents'] = len(user_agents)
#     if action_lines:
#         json_log['action_lines'] = action_lines
#     if text:
#         json_log['size'] = len(text)


# Path: tests/unit/test_robots.py
import pytest
import cocrawler.robots as robots
from cocrawler.robots import is_plausible_robots, robots_facets

    assert robots.strip_bom(robots_txt) == b''
    robots_txt = b'\xef\xbb\xbf  '
    assert robots.strip_bom(robots_txt) == b'  '

    robots_txt = b'\xfe\xff'
    assert robots.strip_bom(robots_txt) == b''
    robots_txt = b'\xfe\xfffoo'
    assert robots.strip_bom(robots_txt) == b'foo'

    robots_txt = b'\xff\xfe'
    assert robots.strip_bom(robots_txt) == b''


def test_is_plausible_robots():
    robots_txt = b'<'
    plausible, message = is_plausible_robots(robots_txt)
    assert not plausible
    assert len(message)

    robots_txt = b''  # application/x-empty
    plausible, message = is_plausible_robots(robots_txt)
    assert plausible
    assert not len(message)

    robots_txt = b'x'*1000001
    plausible, message = is_plausible_robots(robots_txt)
    assert not plausible
    assert len(message)

    robots_txt = b'foo'

    plausible, message = is_plausible_robots(robots_txt)