# Path: breadability/_compat.py
# def to_unicode(object):
#     try:
#         if isinstance(object, unicode):
#             return object
#         elif isinstance(object, bytes):
#             return object.decode("utf8")
#         else:
#             # try decode instance to unicode
#             return instance_to_unicode(object)
#     except UnicodeError:
#         # recover from codec error and use 'repr' function
#         return to_unicode(repr(object))
# 
# Path: breadability/readable.py
# SCORABLE_TAGS = ("div", "p", "td", "pre", "article")
# ANNOTATION_TAGS = (
#     "a", "abbr", "acronym", "b", "big", "blink", "blockquote", "br", "cite",
#     "code", "dd", "del", "dir", "dl", "dt", "em", "font", "h", "h1", "h2",
#     "h3", "h4", "h5", "h6", "hr", "i", "ins", "kbd", "li", "marquee", "menu",
#     "ol", "p", "pre", "q", "s", "samp", "span", "strike", "strong", "sub",
#     "sup", "tt", "u", "ul", "var",
# )
# NULL_DOCUMENT = """
# <html>
#     <head>
#         <meta http-equiv="Content-Type" content="text/html;charset=UTF-8">
#     </head>
#     <body>
#     </body>
# </html>
# """
# def ok_embedded_video(node):
# def build_base_document(dom, return_fragment=True):
# def build_error_document(dom, return_fragment=True):
# def document_from_fragment(fragment, return_fragment):
# def check_siblings(candidate_node, candidate_list):
# def clean_document(node):
# def drop_nodes_with_parents(nodes):
# def clean_conditionally(node):
# def prep_article(doc):
# def find_candidates(document):
# def is_bad_link(node):
#     def __init__(self, html, url=None, return_fragment=True):
#     def __str__(self):
#     def __unicode__(self):
#     def dom(self):
#     def candidates(self):
#     def main_text(self):
#     def readable(self):
#     def readable_dom(self):
#     def _readable(self):
#     def _remove_orphans(self, dom):
#     def _handle_no_candidates(self):
# def leaf_div_elements_into_paragraphs(document):
# class Article(object):
# 
# Path: breadability/scoring.py
# class ScoredNode(object):
#     """
#     We need Scored nodes we use to track possible article matches
# 
#     We might have a bunch of these so we use __slots__ to keep memory usage
#     down.
#     """
#     __slots__ = ('node', 'content_score')
# 
#     def __init__(self, node):
#         """Given node, set an initial score and weigh based on css and id"""
#         self.node = node
#         self.content_score = 0
# 
#         if node.tag in ('div', 'article'):
#             self.content_score = 5
#         if node.tag in ('pre', 'td', 'blockquote'):
#             self.content_score = 3
# 
#         if node.tag in ('address', 'ol', 'ul', 'dl', 'dd', 'dt', 'li', 'form'):
#             self.content_score = -3
#         if node.tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'th'):
#             self.content_score = -5
# 
#         self.content_score += get_class_weight(node)
# 
#     @property
#     def hash_id(self):
#         return generate_hash_id(self.node)
# 
#     def __repr__(self):
#         if self.node is None:
#             return "<NullScoredNode with score {2:0.1F}>" % self.content_score
# 
#         return "<ScoredNode {0} {1}: {2:0.1F}>".format(
#             self.node.tag,
#             self.node.attrib,
#             self.content_score
#         )
# 
# Path: tests/utils.py
# def load_article(file_name):
#     """Helper to fetch in the content of a test article."""
#     file_path = join(TEST_DIR, "data/articles", file_name)
#     with open(file_path, "rb") as file:
#         return file.read()
# 
# def load_snippet(file_name):
#     """Helper to fetch in the content of a test snippet."""
#     file_path = join(TEST_DIR, "data/snippets", file_name)
#     with open(file_path, "rb") as file:
#         return file.read()


import pytest
from lxml.etree import tounicode
from lxml.html import document_fromstring, fragment_fromstring
from breadability._compat import to_unicode
from breadability.readable import (Article, get_class_weight, get_link_density, is_bad_link,
                                   leaf_div_elements_into_paragraphs, score_candidates, )
from breadability.scoring import ScoredNode
from .utils import load_article, load_snippet


def test_we_get_candidates():
    """Processing candidates should get us a list of nodes to try out."""
    doc = document_fromstring(load_article("ars.001.html"))
    test_nodes = tuple(doc.iter("p", "td", "pre"))
    candidates = score_candidates(test_nodes)

    # this might change as we tweak our algorithm, but if it does,
    # it signifies we need to look at what we changed.
    assert len(candidates.keys()) == 37

    # one of these should have a decent score
    scores = sorted(c.content_score for c in candidates.values())
    assert scores[-1] > 100


def test_bonus_score_per_100_chars_in_p():
    """Nodes get 1 point per 100 characters up to max. 3 points."""
    def build_candidates(length):
        html = "<p>%s</p>" % ("c" * length)
        node = fragment_fromstring(html)

        return [node]

    test_nodes = build_candidates(50)
    candidates = score_candidates(test_nodes)
    pscore_50 = max(c.content_score for c in candidates.values())

    test_nodes = build_candidates(100)

# Path: breadability/_compat.py
# def to_unicode(object):
#     try:
#         if isinstance(object, unicode):
#             return object
#         elif isinstance(object, bytes):
#             return object.decode("utf8")
#         else:
#             # try decode instance to unicode
#             return instance_to_unicode(object)
#     except UnicodeError:
#         # recover from codec error and use 'repr' function
#         return to_unicode(repr(object))
# 
# Path: breadability/readable.py
# SCORABLE_TAGS = ("div", "p", "td", "pre", "article")
# ANNOTATION_TAGS = (
#     "a", "abbr", "acronym", "b", "big", "blink", "blockquote", "br", "cite",
#     "code", "dd", "del", "dir", "dl", "dt", "em", "font", "h", "h1", "h2",
#     "h3", "h4", "h5", "h6", "hr", "i", "ins", "kbd", "li", "marquee", "menu",
#     "ol", "p", "pre", "q", "s", "samp", "span", "strike", "strong", "sub",
#     "sup", "tt", "u", "ul", "var",
# )
# NULL_DOCUMENT = """
# <html>
#     <head>
#         <meta http-equiv="Content-Type" content="text/html;charset=UTF-8">
#     </head>
#     <body>
#     </body>
# </html>
# """
# def ok_embedded_video(node):
# def build_base_document(dom, return_fragment=True):
# def build_error_document(dom, return_fragment=True):
# def document_from_fragment(fragment, return_fragment):
# def check_siblings(candidate_node, candidate_list):
# def clean_document(node):
# def drop_nodes_with_parents(nodes):
# def clean_conditionally(node):
# def prep_article(doc):
# def find_candidates(document):
# def is_bad_link(node):
#     def __init__(self, html, url=None, return_fragment=True):
#     def __str__(self):
#     def __unicode__(self):
#     def dom(self):
#     def candidates(self):
#     def main_text(self):
#     def readable(self):
#     def readable_dom(self):
#     def _readable(self):
#     def _remove_orphans(self, dom):
#     def _handle_no_candidates(self):
# def leaf_div_elements_into_paragraphs(document):
# class Article(object):
# 
# Path: breadability/scoring.py
# class ScoredNode(object):
#     """
#     We need Scored nodes we use to track possible article matches
# 
#     We might have a bunch of these so we use __slots__ to keep memory usage
#     down.
#     """
#     __slots__ = ('node', 'content_score')
# 
#     def __init__(self, node):
#         """Given node, set an initial score and weigh based on css and id"""
#         self.node = node
#         self.content_score = 0
# 
#         if node.tag in ('div', 'article'):
#             self.content_score = 5
#         if node.tag in ('pre', 'td', 'blockquote'):
#             self.content_score = 3
# 
#         if node.tag in ('address', 'ol', 'ul', 'dl', 'dd', 'dt', 'li', 'form'):
#             self.content_score = -3
#         if node.tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'th'):
#             self.content_score = -5
# 
#         self.content_score += get_class_weight(node)
# 
#     @property
#     def hash_id(self):
#         return generate_hash_id(self.node)
# 
#     def __repr__(self):
#         if self.node is None:
#             return "<NullScoredNode with score {2:0.1F}>" % self.content_score
# 
#         return "<ScoredNode {0} {1}: {2:0.1F}>".format(
#             self.node.tag,
#             self.node.attrib,
#             self.content_score
#         )
# 
# Path: tests/utils.py
# def load_article(file_name):
#     """Helper to fetch in the content of a test article."""
#     file_path = join(TEST_DIR, "data/articles", file_name)
#     with open(file_path, "rb") as file:
#         return file.read()
# 
# def load_snippet(file_name):
#     """Helper to fetch in the content of a test snippet."""
#     file_path = join(TEST_DIR, "data/snippets", file_name)
#     with open(file_path, "rb") as file:
#         return file.read()


# Path: tests/test_readable.py
import pytest
from lxml.etree import tounicode
from lxml.html import document_fromstring, fragment_fromstring
from breadability._compat import to_unicode
from breadability.readable import (Article, get_class_weight, get_link_density, is_bad_link,
                                   leaf_div_elements_into_paragraphs, score_candidates, )
from breadability.scoring import ScoredNode
from .utils import load_article, load_snippet



def test_we_get_candidates():
    """Processing candidates should get us a list of nodes to try out."""
    doc = document_fromstring(load_article("ars.001.html"))
    test_nodes = tuple(doc.iter("p", "td", "pre"))
    candidates = score_candidates(test_nodes)

    # this might change as we tweak our algorithm, but if it does,
    # it signifies we need to look at what we changed.
    assert len(candidates.keys()) == 37

    # one of these should have a decent score
    scores = sorted(c.content_score for c in candidates.values())
    assert scores[-1] > 100


def test_bonus_score_per_100_chars_in_p():
    """Nodes get 1 point per 100 characters up to max. 3 points."""
    def build_candidates(length):
        html = "<p>%s</p>" % ("c" * length)
        node = fragment_fromstring(html)

        return [node]

    test_nodes = build_candidates(50)
    candidates = score_candidates(test_nodes)
    pscore_50 = max(c.content_score for c in candidates.values())

    test_nodes = build_candidates(100)

    candidates = score_candidates(test_nodes)