<commit_before>from nose.tools import eq_
import bs4

from . import TestCase

import features

def doc(content, url="http://example.com/folder1/somedoc.html"):
  return dict(
    url = url,
    dom = bs4.BeautifulSoup(content),
    payload = content
  )

class TestFeatureExtractors(TestCase):

  def test_scripts(self):
    content =  """
    <html>
    <body>
      <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
      <script src="http://ajax.googleapis.com/ajax/libs/chrome-frame/1.0.3/CFInstall.min.js">
    </body>
    </html>"""

    doc_features =  list(features.script(doc(content)))
    eq_(
      doc_features,
      [('script', u'http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js'), 
      ('script', u'http://ajax.googleapis.com/ajax/libs/chrome-frame/1.0.3/CFInstall.min.js')]
    )

  def test_links(self):
    content = """
    <html>
    <body>
      <a href="http://blah.com">blah</a>
      <a href="../relative/./link">Hi mom</a>
    </body>
    </html>
    """
    f = features.outbound()
    doc_features = list(f(doc(content)))
    self.assertSequenceEqual(
      doc_features,
      [
        ('link_to', 'blah.com'),
      ]
    )

  def test_tags(self):
    content = """
    <html>
    <body>
      <a href="http://blah.com">blah</a>
      <a href="../relative/./link">Hi mom</a>
      <div id="1">
        <span></span>
        <div>
          <ul>
           <li>One</li>
           <li>Two</li>
          </ul>
        </div>
      </div>
    </body>
    </html>
    """
    doc_features = list(features.tag(doc(content)))
    self.assertSequenceEqual(
      doc_features,
      [
        ('tag', 'html'),
        ('tag', 'html>body'),
        ('tag', 'html>body>a'),
        ('tag', 'html>body>a'),
        ('tag', 'html>body>div'),
        ('tag', 'html>body>div>span'),
        ('tag', 'html>body>div>div'),
        ('tag', 'html>body>div>div>ul'),
        ('tag', 'html>body>div>div>ul>li'),
        ('tag', 'html>body>div>div>ul>li')
      ]
    )
   <commit_msg>tags only returns the count now not the hiearchy<commit_after>from nose.tools import eq_
import bs4

from . import TestCase

import features

def doc(content, url="http://example.com/folder1/somedoc.html"):
  return dict(
    url = url,
    dom = bs4.BeautifulSoup(content),
    payload = content
  )

class TestFeatureExtractors(TestCase):

  def test_scripts(self):
    content =  """
    <html>
    <body>
      <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
      <script src="http://ajax.googleapis.com/ajax/libs/chrome-frame/1.0.3/CFInstall.min.js">
    </body>
    </html>"""

    doc_features =  list(features.script(doc(content)))
    eq_(
      doc_features,
      [('script', u'http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js'), 
      ('script', u'http://ajax.googleapis.com/ajax/libs/chrome-frame/1.0.3/CFInstall.min.js')]
    )

  def test_links(self):
    content = """
    <html>
    <body>
      <a href="http://blah.com">blah</a>
      <a href="../relative/./link">Hi mom</a>
    </body>
    </html>
    """
    f = features.outbound()
    doc_features = list(f(doc(content)))
    self.assertSequenceEqual(
      doc_features,
      [
        ('link_to', 'blah.com'),
      ]
    )

  def test_tags(self):
    content = """
    <html>
    <body>
      <a href="http://blah.com">blah</a>
      <a href="../relative/./link">Hi mom</a>
      <div id="1">
        <span></span>
        <div>
          <ul>
           <li>One</li>
           <li>Two</li>
          </ul>
        </div>
      </div>
    </body>
    </html>
    """
    doc_features = list(features.tag(doc(content)))

    self.assertSequenceEqual(
      doc_features,
      [
        ('tag', 'body:1'), 
        ('tag', 'a:2'), 
        ('tag', 'span:1'), 
        ('tag', 'li:2'), 
        ('tag', 'ul:1'), 
        ('tag', 'html:1'), 
        ('tag', 'div:2')
      ]
    )
   
{'fraction_non_alphanumeric': 0.20161290322580644, 'fraction_numerical': 0.009828629032258064, 'mean_word_length': 2.15, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 100, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '80485', 'n_tokens_mistral': 1460, 'n_tokens_neox': 1424, 'n_words': 220}