<commit_before>import re
from collections import defaultdict

import visitors

from legistar.base import Base
from legistar.fields import ElementAccessor


class Visitor(Base, visitors.Visitor):
    '''Visits a detail page and collects all the displayed fields into a
    dictionary that maps label text to taterized DOM nodes.

    Effectively groups different elements and their attributes by the unique
    sluggy part of their verbose aspx id names. For example, the 'Txt' part
    of 'ctl00_contentPlaceholder_lblTxt'.
    '''
    # ------------------------------------------------------------------------
    # These methods customize the visitor.
    # ------------------------------------------------------------------------
    def __init__(self):
        self.data = defaultdict(dict)

    def finalize(self):
        '''Reorganize the data so it's readable labels (viewable on the page)
        are the dictionary keys, instead of the sluggy text present in their
        id attributes. Wrap each value in a DetailField.
        '''
        newdata = {}
        for id_attr, data in tuple(self.data.items()):
            alias = data.get('label', id_attr).strip(':')
            value = self.cfg.make_child(ElementAccessor, data['el'])
            newdata[alias] = value
            if alias != id_attr:
                newdata[id_attr] = value
        return newdata

    def get_nodekey(self, el):
        '''We're visiting a treebie-ized lxml.html document, so dispatch is
        based on the tag attribute.
        '''
        # The weird Comment tag.
        tag = el.tag
        if not isinstance(tag, str):
            raise self.Continue()
        return tag

    def get_children(self, el):
        return tuple(el)

    # ------------------------------------------------------------------------
    # The DOM visitor methods.
    # ------------------------------------------------------------------------
    def visit_a(self, el):
        attrib = el.attrib
        if 'id' not in attrib:
            return

        # If it's a field label, collect the text and href.
        matchobj = re.search(r'_hyp(.+)', attrib['id'])
        if matchobj:
            key = matchobj.group(1)
            data = self.data[key]
            data.update(el=el)
            if 'label' not in data:
                label = el.text_content().strip(':')<commit_msg>Nuke treebie references<commit_after>import re
from collections import defaultdict

import visitors

from legistar.base import Base
from legistar.fields import ElementAccessor


class Visitor(Base, visitors.Visitor):
    '''Visits a detail page and collects all the displayed fields into a
    dictionary that maps label text to DOM nodes.

    Effectively groups different elements and their attributes by the unique
    sluggy part of their verbose aspx id names. For example, the 'Txt' part
    of 'ctl00_contentPlaceholder_lblTxt'.
    '''
    # ------------------------------------------------------------------------
    # These methods customize the visitor.
    # ------------------------------------------------------------------------
    def __init__(self):
        self.data = defaultdict(dict)

    def finalize(self):
        '''Reorganize the data so it's readable labels (viewable on the page)
        are the dictionary keys, instead of the sluggy text present in their
        id attributes. Wrap each value in a DetailField.
        '''
        newdata = {}
        for id_attr, data in tuple(self.data.items()):
            alias = data.get('label', id_attr).strip(':')
            value = self.cfg.make_child(ElementAccessor, data['el'])
            newdata[alias] = value
            if alias != id_attr:
                newdata[id_attr] = value
        return newdata

    def get_nodekey(self, el):
        '''We're visiting an lxml.html doc, so dispatch based on the tag.
        '''
        # The weird Comment tag.
        tag = el.tag
        if not isinstance(tag, str):
            raise self.Continue()
        return tag

    def get_children(self, el):
        return tuple(el)

    # ------------------------------------------------------------------------
    # The DOM visitor methods.
    # ------------------------------------------------------------------------
    def visit_a(self, el):
        attrib = el.attrib
        if 'id' not in attrib:
            return

        # If it's a field label, collect the text and href.
        matchobj = re.search(r'_hyp(.+)', attrib['id'])
        if matchobj:
            key = matchobj.group(1)
            data = self.data[key]
            data.update(el=el)
            if 'label' not in data:
                label = el.text_content().strip(':')
{'fraction_non_alphanumeric': 0.20012760527435133, 'fraction_numerical': 0.0012760527435133986, 'mean_word_length': 1.483104540654699, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4082958', 'n_tokens_mistral': 1183, 'n_tokens_neox': 1137, 'n_words': 409}