<jupyter_start><jupyter_text>Sentence tokenization from Wikipediahttps://dumps.wikimedia.org/Take all the text from jsons, lower-case it, remove all punctuation, and remove leading/trailing whitespaceTokenize the sentences and write them to a file where each line is one sentenceSentences containing numeric characters are excluded To Do[ ] remove single char words excpet of ['a', 'i', 'o', 's', 'u', 'v'][ ] remove lines with non defined in alphabet.txt<jupyter_code>import os
from nltk.tokenize import sent_tokenize
import string
import re
# work_dir = '/workspace/data/wikipedia/'
work_dir = '/home/jakub/Documents/projekt/databazeknih/'


def hasNumbers(inputString):
    return bool(re.search(r'\d', inputString))

translator = str.maketrans('', '', string.punctuation,)


def charFilter (line):
    vysledek = ''
    for ch in line:
        if ch not in ' aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzž':
            ch = ''
        vysledek = vysledek + ch
    
    
    return vysledek
with open(work_dir + 'preprocessed.txt', "w+") as out_file:
    
    for input_file in os.listdir(work_dir):
        print(input_file)
        str_builder = ''
        with open(work_dir + input_file, "r") as in_file:
            for line in in_file:
                words = sent_tokenize(line)

                if len(words) < 2:  # Skip 1 word sentences
                    continue

                
                if hasNumbers(line):
                    continue

                for s in words:
                    str_builder += charFilter(s.lower()).strip() + ' '
                str_builder = str_builder.strip() + '\n'

                if len(str_builder) < 3:
                    continue

        
        out_file.write(str_builder)<jupyter_output>hodina_tance_a_lasky.epub.txt
proces.epub.txt
domaci_kucharka.epub.txt
slava_stroju_a_mest.epub.txt
valka_s_mloky.epub.txt
smrt_krasnych_srncu.txt
hovory_s_t_g_masarykem.epub.txt
oral2013_vert.txt
tanecni_hodiny_pro_starsi_a_pokrocile.epub.txt
hubeny_nas_nedostanou.epub.txt
ceska_inteligence.epub.txt
ctyricet_dnu.epub.txt
urazeni_a_ponizeni.epub.txt
tri_kluci.epub.txt
manzelky_prezidentu.epub.txt
pet_nedel_v_balone.epub.txt
pusinky.epub.txt
rur.epub.txt
zahradnikuv_rok.epub.txt
preprocessed.txt
bajecna_leta_pod_psa.epub.txt
anglicke_listy.epub.txt
kopretiny_pro_zameckou_pani.epub.txt
liska_bystrouska.epub.txt
holky_z_porcelanu.epub.txt
tajemstvi_sametu.epub.txt
lovci_mamutu.epub.txt
mluviti_stribro.epub.txt
santiniho_jazyk.epub.txt
rozmarne_leto.epub.txt
velky_gatsby.epub.txt
hrichy_pro_patera_knoxe.epub.txt
tankovy_prapor.epub.txt
povidky_o_manzelstvi_a_o_sexu.epub.txt
inzerat_na_dum_ve_kterym_uz_nechci_bydlet.epub.txt
plachy_milionar_prichazi.epub.txt
zakladni_pojmy.epub.txt
pan_kdyb[...]
{'fraction_non_alphanumeric': 0.08850199492201669, 'fraction_numerical': 0.0025389916575988395, 'mean_word_length': 3.3364779874213837, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '431735', 'license': 'Apache-2.0', 'n_tokens_mistral': 1109, 'n_tokens_neox': 1073, 'n_words': 184, 'path': 'language_model/wikipedia.ipynb', 'repo_name': 'slesinger/czech-speech-to-text', 'size': '2757'}