<commit_before>#!/usr/bin/env python

import os, glob
from plasTeX import Command, Environment, TheCounter

def ProcessOptions(options, document):
    context = document.context

    # Lists
    context.newcounter('enumi')
    context.newcounter('enumii', resetby='enumi')
    context.newcounter('enumiii', resetby='enumii')
    context.newcounter('enumiv', resetby='enumiii')

    # Sections
    context.newcounter('part', resetby='volume', 
                       format='$part')
    context.newcounter('chapter', resetby='part', 
                       format='$chapter')
    context.newcounter('section', resetby='chapter', 
                       format='${thechapter}.${section}')
    context.newcounter('subsection', resetby='section', 
                       format='${thesection}.${subsection}')
    context.newcounter('subsubsection', resetby='subsection', 
                       format='${thesubsection}.${subsubsection}')
    context.newcounter('paragraph', resetby='subsubsection', 
                       format='${thesubsubsection}.${paragraph}')
    context.newcounter('subparagraph', resetby='paragraph',
                       format='${theparagraph}.${subparagraph}')
    context.newcounter('subsubparagraph', resetby='subparagraph',
                       format='${thesubparagraph}.${subsubparagraph}')

    context.newcounter('equation', resetby='chapter',
                       format='${thechapter}.${equation}')

    context.newcounter('secnumdepth')
    context.newcounter('tocdepth')
    context.newcounter('page')

    # Floats
    context.newcounter('figure', resetby='chapter', 
                       format='${thechapter}.${figure}')
    context.newcounter('table', resetby='chapter',
                       format='${thechapter}.${table}')
    context.newcounter('topnumber')
    context.newcounter('bottomnumber')
    context.newcounter('totalnumber')
    context.newcounter('dbltopnumber')

    language = False
    from plasTeX.Base.LaTeX import Languages
    languages = glob.glob(os.path.join(os.path.dirname(Languages.__file__),'*.py'))
    languages = [os.path.splitext(os.path.basename(x))[0] for x in languages]
    for key, value in options.items():
        if key == 'language':
            language = True
            context.loadLanguage(value, document)
        elif key in languages:
            language = True
            context.loadLanguage(key, document)

    if not language:
        context.loadLanguage('american', document)<commit_msg>Changes for new XML based language terms<commit_after>#!/usr/bin/env python

import os, glob
from plasTeX import Command, Environment, TheCounter

def ProcessOptions(options, document):
    context = document.context

    # Lists
    context.newcounter('enumi')
    context.newcounter('enumii', resetby='enumi')
    context.newcounter('enumiii', resetby='enumii')
    context.newcounter('enumiv', resetby='enumiii')

    # Sections
    context.newcounter('part', resetby='volume', 
                       format='$part')
    context.newcounter('chapter', resetby='part', 
                       format='$chapter')
    context.newcounter('section', resetby='chapter', 
                       format='${thechapter}.${section}')
    context.newcounter('subsection', resetby='section', 
                       format='${thesection}.${subsection}')
    context.newcounter('subsubsection', resetby='subsection', 
                       format='${thesubsection}.${subsubsection}')
    context.newcounter('paragraph', resetby='subsubsection', 
                       format='${thesubsubsection}.${paragraph}')
    context.newcounter('subparagraph', resetby='paragraph',
                       format='${theparagraph}.${subparagraph}')
    context.newcounter('subsubparagraph', resetby='subparagraph',
                       format='${thesubparagraph}.${subsubparagraph}')

    context.newcounter('equation', resetby='chapter',
                       format='${thechapter}.${equation}')

    context.newcounter('secnumdepth')
    context.newcounter('tocdepth')
    context.newcounter('page')

    # Floats
    context.newcounter('figure', resetby='chapter', 
                       format='${thechapter}.${figure}')
    context.newcounter('table', resetby='chapter',
                       format='${thechapter}.${table}')
    context.newcounter('topnumber')
    context.newcounter('bottomnumber')
    context.newcounter('totalnumber')
    context.newcounter('dbltopnumber')

    language = False
    languages = document.context.languages.keys()
    for key, value in options.items():
        if key == 'language':
            language = True
            context.loadLanguage(value, document)
        elif key in languages:
            language = True
            context.loadLanguage(key, document)
{'fraction_non_alphanumeric': 0.14211076280041798, 'fraction_numerical': 0.0002089864158829676, 'mean_word_length': 2.8942229454841333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2849789', 'n_tokens_mistral': 1274, 'n_tokens_neox': 1218, 'n_words': 200}