<commit_before>import copy
import os
import pickle
import re

from lxml import etree

from regparser import api_writer, content
from regparser.federalregister import fetch_notice_json, fetch_notices
from regparser.history.notices import (
    applicable as applicable_notices, group_by_eff_date)
from regparser.history.delays import modify_effective_dates
from regparser.layer import (
    external_citations, formatting, graphics, key_terms, internal_citations,
    interpretations, meta, paragraph_markers, section_by_section,
    table_of_contents, terms)
from regparser.notice.compiler import compile_regulation
from regparser.tree import struct
# from regparser.tree.build import build_whole_regtree
from regparser.tree.xml_parser import reg_text


class Builder(object):
    """Methods used to build all versions of a single regulation, their
    layers, etc. It is largely glue code"""

    def __init__(self, cfr_title, cfr_part, doc_number, checkpointer=None):
        self.cfr_title = cfr_title
        self.cfr_part = cfr_part
        self.doc_number = doc_number
        self.checkpointer = checkpointer or NullCheckpointer()
        self.writer = api_writer.Client()

        self.eff_notices = self.checkpointer.checkpoint(
            "notices",
            lambda: notices_for_cfr_part(self.cfr_title, self.cfr_part)
        )
        self.notices = []
        for notice_group in self.eff_notices.values():
            self.notices.extend(notice_group)

    def write_notices(self):
        for notice in self.notices:
            #  No need to carry this around
            del notice['meta']
            self.writer.notice(notice['document_number']).write(notice)

    def write_regulation(self, reg_tree):
        self.writer.regulation(self.cfr_part, self.doc_number).write(reg_tree)

    def gen_and_write_layers(self, reg_tree, act_info, cache, notices=None):
        if notices is None:
            notices = applicable_notices(self.notices, self.doc_number)
        for ident, layer_class in (
                ('external-citations',
                    external_citations.ExternalCitationParser),
                ('meta', meta.Meta),
                ('analyses', section_by_section.SectionBySection),
                ('internal-citations',<commit_msg>New tag to be backwards compatible<commit_after>import copy
import os
import pickle
import re

from lxml import etree

from regparser import api_writer, content
from regparser.federalregister import fetch_notice_json, fetch_notices
from regparser.history.notices import (
    applicable as applicable_notices, group_by_eff_date)
from regparser.history.delays import modify_effective_dates
from regparser.layer import (
    external_citations, formatting, graphics, key_terms, internal_citations,
    interpretations, meta, paragraph_markers, section_by_section,
    table_of_contents, terms)
from regparser.notice.compiler import compile_regulation
from regparser.tree import struct
# from regparser.tree.build import build_whole_regtree
from regparser.tree.xml_parser import reg_text


class Builder(object):
    """Methods used to build all versions of a single regulation, their
    layers, etc. It is largely glue code"""

    def __init__(self, cfr_title, cfr_part, doc_number, checkpointer=None):
        self.cfr_title = cfr_title
        self.cfr_part = cfr_part
        self.doc_number = doc_number
        self.checkpointer = checkpointer or NullCheckpointer()
        self.writer = api_writer.Client()

        self.eff_notices = self.checkpointer.checkpoint(
            "effective-notices",
            lambda: notices_for_cfr_part(self.cfr_title, self.cfr_part)
        )
        self.notices = []
        for notice_group in self.eff_notices.values():
            self.notices.extend(notice_group)

    def write_notices(self):
        for notice in self.notices:
            #  No need to carry this around
            del notice['meta']
            self.writer.notice(notice['document_number']).write(notice)

    def write_regulation(self, reg_tree):
        self.writer.regulation(self.cfr_part, self.doc_number).write(reg_tree)

    def gen_and_write_layers(self, reg_tree, act_info, cache, notices=None):
        if notices is None:
            notices = applicable_notices(self.notices, self.doc_number)
        for ident, layer_class in (
                ('external-citations',
                    external_citations.ExternalCitationParser),
                ('meta', meta.Meta),
                ('analyses', section_by_section.SectionBySection),
                ('internal-citations',
{'fraction_non_alphanumeric': 0.0770237835478944, 'fraction_numerical': 0.0, 'mean_word_length': 3.224884792626728, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4186205', 'n_tokens_mistral': 1322, 'n_tokens_neox': 1261, 'n_words': 318}