<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#import string
import re
import bgdict
import csv
import os

(bg_en, en_bg, bg_bg, enWikt) = bgdict.buildDicts()

def makeFreqFromText(text, usedWords):
    wordBoundry = re.compile('\W+',re.UNICODE)

    freqDict = {}
    # TODO: Fix capitals for names
    for w in wordBoundry.split(text):
        if w != w.lower():
            # Detect real words that are capitals
            w = w.lower()
        if freqDict.has_key(w):
            freqDict[w] += 1
        else:
            freqDict[w] = 1
    for common in list(set(usedWords).intersection(freqDict.keys())):
        #print common.encode('utf-8')
        del freqDict[common]
    return freqDict

def createChapterFile(filename,freqDict):
    if os.path.dirname(filename):
        if not os.path.isdir(os.path.dirname(filename)):
            os.mkdir(os.path.dirname(filename))
    csvWriter = csv.writer(open(filename + '.csv', 'wb'))

    for tup in sorted(sorted(freqDict.items(), key=lambda x: x[0].encode('utf-8')), key=lambda x: x[1], reverse=True):
        key = tup[0].encode('utf-8')
        value = lookupWord(tup[0])
        csvWriter.writerow([key , value])

def lookupWord(word):
    try:
        return enWikt[word]
    except:
        try:
            return bg_en[bg_bg[word]].encode('utf-8')
        except:
            try:
                return bg_en[word].encode('utf-8')
            except:
                try:
                    return bg_en[word.title()].encode('utf-8')
                except:
                    return "Unknown"<commit_msg>fix wordlist and add some debugging, still have to find a better Slovoformi mapping<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#import string
import re
import bgdict
import csv
import os

(bg_en,en_bg,bg_bg,bg_enWikt, en_bgWikt) = bgdict.buildDicts()

def makeFreqFromText(text, usedWords):
    wordBoundry = re.compile('\W+',re.UNICODE)

    freqDict = {}
    # TODO: Fix capitals for names
    for w in wordBoundry.split(text):
        if w != w.lower():
            # Detect real words that are capitals
            w = w.lower()
        if freqDict.has_key(w):
            freqDict[w] += 1
        else:
            freqDict[w] = 1
    for common in list(set(usedWords).intersection(freqDict.keys())):
        #print common.encode('utf-8')
        del freqDict[common]
    return freqDict

def createChapterFile(filename,freqDict):
    if os.path.dirname(filename):
        if not os.path.isdir(os.path.dirname(filename)):
            os.mkdir(os.path.dirname(filename))
    csvWriter = csv.writer(open(filename + '.csv', 'wb'))

    for tup in sorted(sorted(freqDict.items(), key=lambda x: x[0].encode('utf-8')), key=lambda x: x[1], reverse=True):
        key = tup[0].encode('utf-8')
        value = lookupWord(tup[0])
        csvWriter.writerow([key , value])

def lookupWord(word):
    try:
        return bg_enWikt[word]
    except:
        print word + " not in bg_enWikt"
        try:
            return bg_enWikt[bg_bg[word]]
        except:
            print word + " not in bg_bg and bg_enWikt"
            try:
                return bg_en[bg_bg[word]].encode('utf-8')
            except:
                print word + " not in bg_bg and bg_en"
                try:
                    return bg_en[word].encode('utf-8')
                except:
                    try:
                        return bg_en[word.title()].encode('utf-8')
                    except:
                        return "Unknown"
{'fraction_non_alphanumeric': 0.13017583664208734, 'fraction_numerical': 0.00737379466817924, 'mean_word_length': 2.064291920069505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3566387', 'n_tokens_mistral': 1167, 'n_tokens_neox': 1119, 'n_words': 255}