<jupyter_start><jupyter_text>CHOMP v2__Misc. Utilities____by <NAME>__  __Last updated November 30__, __2021__  [https://github.com/seangilleran/chomp2](https://github.com/seangilleran/chomp2)  Check Language (With Spacy)<jupyter_code>import os

import en_core_web_sm
from spacy.language import Language
from spacy_langdetect import LanguageDetector

path = "./corpus"


# Load language detector.
@Language.factory("language_detector")
def language_detector(nlp, name):
    return LanguageDetector()
nlp = en_core_web_sm.load()
nlp.add_pipe("language_detector", last=True)

for file in [f for f in os.listdir(path) if f.endswith(f".txt")]:

    with open(os.path.join(path, file), "r", encoding="utf-8") as f:
        text = f.read()

    # Check language.
    lang = nlp(text)._.language
    language = lang["language"]
    score = lang["score"]

    print(f"{language.capitalize()} ({(score * 100):.0f}%): {file}")
    with open("lang_check.csv", "a", encoding="utf-8") as f:
        f.write(f"{language},{score},{file}\n")<jupyter_output><empty_output><jupyter_text>Check Language (With Tag)<jupyter_code>import json
import os

path = "./meta"

for file in [f for f in os.listdir(path) if f.endswith(".json")]:

    with open(os.path.join(path, file), "r", encoding="utf-8") as f:
        collection = json.loads(f.read())

    for item in collection["items"]:
        lang = item["language"]

        for file in item["files"]:
            with open("lang_check.csv", "a", encoding="utf-8") as f:
                f.write(f"{lang},{file['name']},{file['id']}\n")<jupyter_output><empty_output><jupyter_text>Convert PDF to TXT<jupyter_code># TODO<jupyter_output><empty_output><jupyter_text>Check Words vs. Enchant Spellcheck<jupyter_code>import os

from nltk.tokenize import word_tokenize
import regex as re

path = ".\corpus"
files = []
words = set()

az = re.compile(r"^[a-zA-Z]+$")


files = [f for f in os.listdir(path) if f.endswith(".txt")]

print(f"Loading {len(files)} files...")

for file in files:
    word_count = len(words)
    with open(os.path.join(path, file), "r", encoding="utf-8") as f:
        text = f.read()
    for word in word_tokenize(text):
        if not az.search(word):
            continue
        words.add(word)
    print(f"Added {len(words) - word_count} words from {file}.")

print("\n** DONE! **")
print(f"Found {len(words)} unique words.")
import enchant

d = enchant.Dict("en_US")


words = list(words)
words.sort()
print(f"Checking {len(words)} words...")

ok_count = 0
nf_count = 0

for word in words:
    print(f"(?) {word} ...", end="")
    if not d.check(word):
        with open("words.txt", "a", encoding="utf-8") as f:
            f.write(f"{word}\n")
        print("Not Found")
        continue
    print("OK")

total = ok_count + nf_count
print("\n** DONE! **")
print(f"Could not find spelling for {nf_count} words out of {total} (corpus {(ok_count/total):.0f}% ok).")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17017663043478262, 'fraction_numerical': 0.007472826086956522, 'mean_word_length': 3.89202657807309, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '501036', 'license': 'MIT', 'n_tokens_mistral': 1088, 'n_tokens_neox': 1057, 'n_words': 246, 'path': '_utilities.ipynb', 'repo_name': 'seangilleran/chomp2', 'size': '2951'}