<commit_before>"""
An oEmbed_ client library.

.. _oEmbed: http://oembed.com/
"""

import contextlib
import HTMLParser
import json
import urllib
import urllib2
import xml.sax
import xml.sax.handler


__all__ = (
    'get_oembed',
)


class LinkExtractor(HTMLParser.HTMLParser):
    """
    A simple subclass of `HTMLParser` for extracting <link> tags from the
    header of a HTML document.
    """

    def __init__(self):
        HTMLParser.HTMLParser.__init__(self)
        self.active = False
        self.finished = False
        self.collected = []

    def handle_starttag(self, tag, attrs):
        if tag == 'head':
            self.active = True
        elif self.active and tag == 'link':
            self.collected.append(dict(attrs))

    def handle_endtag(self, tag):
        if tag == 'head':
            self.active = False
            self.finished = True

    @staticmethod
    def extract(fh):
        parser = LinkExtractor()
        with contextlib.closing(parser):
            while not parser.finished:
                chunk = fh.read(2048)
                if chunk == '':
                    break
                parser.feed(chunk)
        return parser.collected


class OEmbedContentHandler(xml.sax.handler.ContentHandler):
    """
    Pulls the fields out of an XML oEmbed document.
    """

    def __init__(self):
        xml.sax.handler.ContentHandler.__init__(self)
        self.current_field = None
        self.current_value = None
        self.depth = 0
        self.fields = {}

    def startElement(self, name, attrs):
        self.depth += 1
        if self.depth == 2:
            self.current_field = name
            self.current_value = ''

    def endElement(self, name):
        if self.depth == 2:
            self.fields[self.current_field] = self.current_value
        self.depth -= 1

    def characters(self, content):
        if self.depth == 2:
            self.current_value += content


def fetch_links(url):
    """
    Extract the <link> tags from the HTML document at the given URL.
    """
    fh = urllib2.urlopen(url)<commit_msg>Only pull in valid fields.<commit_after>"""
An oEmbed_ client library.

.. _oEmbed: http://oembed.com/
"""

import contextlib
import HTMLParser
import json
import urllib
import urllib2
import xml.sax
import xml.sax.handler


__all__ = (
    'get_oembed',
)


class LinkExtractor(HTMLParser.HTMLParser):
    """
    A simple subclass of `HTMLParser` for extracting <link> tags from the
    header of a HTML document.
    """

    def __init__(self):
        HTMLParser.HTMLParser.__init__(self)
        self.active = False
        self.finished = False
        self.collected = []

    def handle_starttag(self, tag, attrs):
        if tag == 'head':
            self.active = True
        elif self.active and tag == 'link':
            self.collected.append(dict(attrs))

    def handle_endtag(self, tag):
        if tag == 'head':
            self.active = False
            self.finished = True

    @staticmethod
    def extract(fh):
        parser = LinkExtractor()
        with contextlib.closing(parser):
            while not parser.finished:
                chunk = fh.read(2048)
                if chunk == '':
                    break
                parser.feed(chunk)
        return parser.collected


class OEmbedContentHandler(xml.sax.handler.ContentHandler):
    """
    Pulls the fields out of an XML oEmbed document.
    """

    valid_fields = set([
        'type', 'version', 'title', 'cache_age',
        'author_name', 'author_url',
        'provider_name', 'provider_url',
        'thumbnail_url', 'thumbnail_width', 'thumbnail_height',
    ])

    def __init__(self):
        xml.sax.handler.ContentHandler.__init__(self)
        self.current_field = None
        self.current_value = None
        self.depth = 0
        self.fields = {}

    def startElement(self, name, attrs):
        self.depth += 1
        if self.depth == 2:
            self.current_field = name
            self.current_value = ''

    def endElement(self, name):
        if self.depth == 2 and self.current_field in self.valid_fields:
            self.fields[self.current_field] = self.current_value
        self.depth -= 1

    def characters(self, content):
        if self.depth == 2:
            self.current_value += content


def fetch_links(url):
    """
    Extract the <link> tags from the HTML document at the given URL.
    """
    fh = urllib2.urlopen(url)
{'fraction_non_alphanumeric': 0.10522773165806597, 'fraction_numerical': 0.0053847879739735245, 'mean_word_length': 2.0744827586206895, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1827010', 'n_tokens_mistral': 1345, 'n_tokens_neox': 1281, 'n_words': 338}