<commit_before>"""
.. module hypallage.microdata
"""
# NOTE The SPEC for HTML5 microdata lives at
# http://www.whatwg.org/specs/web-apps/current-work/multipage/microdata.html
# To resolve URLs in comments below, substitute this URL for SPEC.

import bs4
from bs4 import BeautifulSoup as bs
import os.path as path
try:  # python 3
    from urllib.request import urlopen
except ImportError:  # python 2
    from urllib2 import urlopen


class Extractor(object):
    """HTML 5 microdata extractor"""

    def __init__(self, extract_from):
        """Constructor takes a file path or URL"""
        self._items = None
        if isinstance(extract_from, bs4.BeautifulSoup):
            self.document = extract_from

        elif path.exists(extract_from):
            with open(extract_from) as thefile:
                rawhtml = thefile.read()
            self.document = bs(rawhtml)

        else:
            with urlopen(extract_from) as thefile:
                rawhtml = thefile.read()
            self.document = bs(rawhtml)

    @property
    def items(self):
        if self._items is None:
            self._items = self._extract_items()
        return self._items

    def to_json(self, **kwargs):
        import json
        # To be compliant with SPEC#extracting-json, we must have no whitespace
        # between tokens by default. The default json separator includes
        # spaces, so we need to replace them. But the caller can override this
        # if they don't care about the spec.
        if 'separators' not in kwargs and 'indent' not in kwargs:
            kwargs['separators'] = (',', ':')

        # To ease comparisons, sort the keys by default. My reading of
        # SPEC#extracting-json allows this, as order of properties is not
        # specified.
        if 'sort_keys' not in kwargs:
            kwargs['sort_keys'] = True

        # TODO Use our own JSONEncoder that knows how to deal with URLs and
        # the various datetime types.
        return json.dumps({'items': self.items}, **kwargs)

    def is_itemscope(self, node):<commit_msg>docstrings<commit_after>"""
.. module hypallage.microdata
"""
# NOTE The SPEC for HTML5 microdata lives at
# http://www.whatwg.org/specs/web-apps/current-work/multipage/microdata.html
# To resolve URLs in comments below, substitute this URL for SPEC.

import bs4
from bs4 import BeautifulSoup as bs
import os.path as path
try:  # python 3
    from urllib.request import urlopen
except ImportError:  # python 2
    from urllib2 import urlopen


class Extractor(object):
    """HTML 5 microdata extractor"""

    def __init__(self, extract_from):
        """Constructor takes a file path, URL, or BeautifulSoup instance."""
        self._items = None
        if isinstance(extract_from, bs4.BeautifulSoup):
            self.document = extract_from

        elif path.exists(extract_from):
            with open(extract_from) as thefile:
                rawhtml = thefile.read()
            self.document = bs(rawhtml)

        else:
            with urlopen(extract_from) as thefile:
                rawhtml = thefile.read()
            self.document = bs(rawhtml)

    @property
    def items(self):
        """
        A list of the extracted items.
        """
        if self._items is None:
            self._items = self._extract_items()
        return self._items

    def to_json(self, **kwargs):
        """
        Returns a serialized JSON string.

        The method accepts any key-word arguments accepted by
        :py:`json.dumps`.

        By default, behaves according to the HTML5 specification, returning
        the most compact form possible. To ease string comparisons, also
        returns the keys sorted. You can override these defaults by passing
        kwargs.
        """
        import json
        # To be compliant with SPEC#extracting-json, we must have no whitespace
        # between tokens by default. The default json separator includes
        # spaces, so we need to replace them. But the caller can override this
        # if they don't care about the spec.
        if 'separators' not in kwargs and 'indent' not in kwargs:
            kwargs['separators'] = (',', ':')

        # To ease comparisons, sort the keys by default. My reading of
        # SPEC#extracting-json allows this, as order of properties is not
        # specified.
        if 'sort_keys' not in kwargs:
            kwargs['sort_keys'] = True

        # TODO Use our own JSONEncoder that knows how to deal with URLs and
        # the various datetime types.
        return json.dumps({'items': self.items}, **kwargs)

    def is_itemscope(self, node):
{'fraction_non_alphanumeric': 0.08234533304591507, 'fraction_numerical': 0.0036645828842422935, 'mean_word_length': 2.3550253073029648, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1512780', 'n_tokens_mistral': 1296, 'n_tokens_neox': 1234, 'n_words': 472}