<commit_before>#!/usr/bin/env python3
# This Python file uses the following encoding: utf-8
from kural import Thirukkural
from tamil.utf8 import get_letters, get_tamil_words, total_maaththirai
from collections import Counter, OrderedDict
from pprint import pprint
import matplotlib.pyplot as plt
import matplotlib

def main():
    eq = Counter()
    eqd = {}
    kural = Thirukkural()
    for kural_no in range(1330):
        kural_words = get_tamil_words(get_letters(kural.get_kural_no(kural_no+1).ta))
        mathirai = sum([total_maaththirai(word) for word in kural_words])
        if eq[mathirai] == 0:
                eqd[mathirai] = [kural_no+1]
        else:
            eqd[mathirai].append(kural_no+1)
        eq[mathirai] += 1
    eq_sorted=OrderedDict(sorted(eq.items(),key=lambda x: x))
    pprint(eq_sorted)
    pprint(eq_sorted.values())
    pprint(eqd)
    print("total = ",sum(eq.values()))
    plt.scatter(eq_sorted.keys(),eq_sorted.values())
    plt.title(u'குறள் மாத்திரை வரிசை',{'fontname':'Catamaran'})
    plt.ylabel(u'குறட்பாக்கள் எண்ணிக்கை',{'fontname':'Catamaran'})
    plt.xlabel(u'மாத்திரை அளவு',{'fontname':'Catamaran'}) #Arial Unicode MS'})
    plt.show()

if __name__ == "__main__":
    main()<commit_msg>Kural Gaussian fit: yields mean=29.5 mathirai and std=+/-2.5 after Gaussian curve fit<commit_after>#!/usr/bin/env python3
# This Python file uses the following encoding: utf-8
from kural import Thirukkural
from tamil.utf8 import get_letters, get_tamil_words, total_maaththirai
from collections import Counter, OrderedDict
from pprint import pprint
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
from scipy.optimize import curve_fit

# Define model function to be used to fit to the data above:
def gauss(x, *p):
    A, mu, sigma = p
    return A*np.exp(-(x-mu)**2/(2.*sigma**2))

def main():
    eq = Counter()
    eqd = {}
    kural = Thirukkural()
    for kural_no in range(1330):
        kural_words = get_tamil_words(get_letters(kural.get_kural_no(kural_no+1).ta))
        mathirai = sum([total_maaththirai(word) for word in kural_words])
        if eq[mathirai] == 0:
                eqd[mathirai] = [kural_no+1]
        else:
            eqd[mathirai].append(kural_no+1)
        eq[mathirai] += 1
    eq_sorted=OrderedDict(sorted(eq.items(),key=lambda x: x))
    pprint(eq_sorted)
    pprint(eq_sorted.values())
    pprint(eqd)
    print("total = ",sum(eq.values()))
    plt.scatter(eq_sorted.keys(),eq_sorted.values())
    plt.ylabel(u'குறட்பாக்கள் எண்ணிக்கை',{'fontname':'Catamaran'})
    plt.xlabel(u'மாத்திரை அளவு',{'fontname':'Catamaran'}) #Arial Unicode MS'})

    # p0 is the initial guess for the fitting coefficients (A, mu and sigma above)
    p0 = [75., 20., 5.]
    coeff, var_matrix = curve_fit(gauss, list(eq_sorted.keys()), list(eq_sorted.values()), p0=p0)

    # Get the fitted curve
    hist_fit = gauss(list(eq_sorted.keys()), *coeff)
    plt.plot(eq_sorted.keys(), hist_fit, label='Gaussian Fitted data (mean=%g, std=%g)'%(coeff[1],coeff[2]))
    plt.title(r'குறள் மாத்திரை வரிசை (Gauss \mu=%g, \sigma=%g)'%(coeff[1],coeff[2]),{'fontname':'Catamaran'})

    # Finally, lets get the fitting parameters, i.e. the mean and standard deviation:
    print ('Fitted mean = ', coeff[1])
    print('Fitted standard deviation = ', coeff[2])

    plt.show()


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.15256752745621846, 'fraction_numerical': 0.013950727218759276, 'mean_word_length': 3.9558823529411766, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4522635', 'n_tokens_mistral': 1351, 'n_tokens_neox': 1348, 'n_words': 274}