<commit_before>#!/usr/bin/env python
"""HTML tag annotator"""

from HTMLParser import HTMLParser

from annotator import *

import re

# create a subclass and override the handler methods
class HTMLOffsetParser(HTMLParser):

    text = ''
    index = 0

    space_chars = set([' ', '\n', '\t'])

    def __init__(self, tagset=None):
        HTMLParser.__init__(self)
        if tagset is not None:
            self.tagset = [tag.lower() for tag in tagset]
        else:
            self.tagset = []
        self.pending_tags = []
        self.tags = []

    def handle_starttag(self, tag, attrs):
        if tag in self.tagset:
            self.pending_tags.append(
                {'name': tag,
                 'start': self.index,
                 'attrs': attrs
                 })

    def handle_endtag(self, tag):
        if self.pending_tags[-1]['name'] == tag:
            pending_tag = self.pending_tags.pop()
            pending_tag['end'] = self.index
            self.tags.append(pending_tag)
        else:
            # Do we want to raise an error or log unexpected end tags?
            pass

    def handle_data(self, data):

        # If we are starting a paragraph and we don't have whitespace after
        # the end of the last paragraph, add a space.
        if (self.pending_tags and<commit_msg>check if pending_tags has any members before trying to access one<commit_after>#!/usr/bin/env python
"""HTML tag annotator"""

from HTMLParser import HTMLParser

from annotator import *

import re

# create a subclass and override the handler methods
class HTMLOffsetParser(HTMLParser):

    text = ''
    index = 0

    space_chars = set([' ', '\n', '\t'])

    def __init__(self, tagset=None):
        HTMLParser.__init__(self)
        if tagset is not None:
            self.tagset = [tag.lower() for tag in tagset]
        else:
            self.tagset = []
        self.pending_tags = []
        self.tags = []

    def handle_starttag(self, tag, attrs):
        if tag in self.tagset:
            self.pending_tags.append(
                {'name': tag,
                 'start': self.index,
                 'attrs': attrs
                 })

    def handle_endtag(self, tag):
        if self.pending_tags and self.pending_tags[-1]['name'] == tag:
            pending_tag = self.pending_tags.pop()
            pending_tag['end'] = self.index
            self.tags.append(pending_tag)
        else:
            # Do we want to raise an error or log unexpected end tags?
            pass

    def handle_data(self, data):

        # If we are starting a paragraph and we don't have whitespace after
        # the end of the last paragraph, add a space.
        if (self.pending_tags and
{'fraction_non_alphanumeric': 0.10081240768094535, 'fraction_numerical': 0.0014771048744460858, 'mean_word_length': 1.9574235807860263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '101468', 'n_tokens_mistral': 781, 'n_tokens_neox': 752, 'n_words': 245}