<commit_before>import pprint
pp = pprint.PrettyPrinter(indent=4)

import re
import os
from wikichatter import SignatureUtils
from wikichatter import WikiIndentUtils
import mwparserfromhell as mwp
from wikichatter import WikiComments as wc
from wikichatter import TextBlocks as tb
from wikichatter import IndentTree as indT
from wikichatter import TalkPageParser

def switch_marker(block_marker = ""):
    if block_marker == "#### ":
        return "     "
    else:
        return "#### "

talk_samples_base = "../talk_samples/"
talk_files = []
for (name, directories, files) in os.walk(talk_samples_base):
    talk_files.extend([name + "/" +f for f in files])



sig_focused = True
for f_path in talk_files:
    print("\n"*3 + f_path + "\n"*3)
    with open(f_path, "r") as f:
        text = f.read()
        indent_tree_root = TalkPageParser.parse(text)
        print(indent_tree_root)<commit_msg>removing dead code from example.py<commit_after>import pprint
pp = pprint.PrettyPrinter(indent=4)

import re
import os
from wikichatter import SignatureUtils
from wikichatter import WikiIndentUtils
import mwparserfromhell as mwp
from wikichatter import WikiComments as wc
from wikichatter import TextBlocks as tb
from wikichatter import IndentTree as indT
from wikichatter import TalkPageParser

talk_samples_base = "./talk_samples/"
talk_files = []
for (name, directories, files) in os.walk(talk_samples_base):
    talk_files.extend([name + "/" +f for f in files])



sig_focused = True
for f_path in talk_files:
    print("\n"*3 + f_path)
    with open(f_path, "r") as f:
        text = f.read()
        indent_tree_root = TalkPageParser.parse(text)
        print(indent_tree_root)
{'fraction_non_alphanumeric': 0.08977407847800238, 'fraction_numerical': 0.002972651605231867, 'mean_word_length': 4.178461538461539, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '40989', 'n_tokens_mistral': 561, 'n_tokens_neox': 541, 'n_words': 161}