<commit_before>#!/usr/bin/env python

from pprint import pprint
from IPython.Shell import IPShellEmbed
import xml.etree.ElementTree
import xml.dom.minidom
import sys
import re
import pickle

ipshell = IPShellEmbed()

fh = open("enwiktionary-latest-pages-meta-current.xml")

articles = {}

read = False

bulRE = re.compile("[bB]ulgarian", re.UNICODE)
#crylRE = re.compile("[\u0400-\u04FF\u0500-\u052F]", re.UNICODE)

while 1:
    line = fh.readline()
    if not line:
        break
    if line == "  <page>\n":
        article = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>"
        read = True
    elif line == "  </page>\n":
        article += line
        read = False
        root = xml.dom.minidom.parseString(article)
        if len(root.getElementsByTagName("text")[0].childNodes) > 0:
            title = root.getElementsByTagName("title")[0].firstChild.data
            text = root.getElementsByTagName("text")[0].firstChild.data
            #if crylRE.search(title):
            #    articles[title] = text
            if bulRE.search(text):
                articles[title] = text
    if read:
        article += line

enWiktBG = open("enWiktBG.pickle",'wb')

pickle.dump(articles, enWiktBG)<commit_msg>move to cpickle and use the highest protocol available<commit_after>#!/usr/bin/env python

from pprint import pprint
from IPython.Shell import IPShellEmbed
import xml.etree.ElementTree
import xml.dom.minidom
import sys
import re
import cPickle as pickle

ipshell = IPShellEmbed()

fh = open("enwiktionary-latest-pages-meta-current.xml")

articles = {}

read = False

bulRE = re.compile("[bB]ulgarian", re.UNICODE)
#crylRE = re.compile("[\u0400-\u04FF\u0500-\u052F]", re.UNICODE)

while 1:
    line = fh.readline()
    if not line:
        break
    if line == "  <page>\n":
        article = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>"
        read = True
    elif line == "  </page>\n":
        article += line
        read = False
        root = xml.dom.minidom.parseString(article)
        if len(root.getElementsByTagName("text")[0].childNodes) > 0:
            title = root.getElementsByTagName("title")[0].firstChild.data
            text = root.getElementsByTagName("text")[0].firstChild.data
            #if crylRE.search(title):
            #    articles[title] = text
            if bulRE.search(text):
                articles[title] = text
    if read:
        article += line

enWiktBG = open("enWiktBG.pickle",'wb')

pickle.dump(articles, enWiktBG, pickle.HIGHEST_PROTOCOL)
{'fraction_non_alphanumeric': 0.13914434226309477, 'fraction_numerical': 0.01679328268692523, 'mean_word_length': 2.72876304023845, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 2, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1403035', 'n_tokens_mistral': 885, 'n_tokens_neox': 845, 'n_words': 174}