<commit_before># -*- coding: utf-8 -*-
"""
@navrajnarula
"""

<commit_msg>attempt k fold cross val<commit_after># -*- coding: utf-8 -*-
"""
@navrajnarula
http://thelillysblog.com/2017/08/18/machine-learning-k-fold-validation/
"""
import nltk # needed for Naive-Bayes
import numpy as np
from sklearn.model_selection import KFold

import pandas
from pandas import read_csv
from pandas import datetime

data = read_csv('data/130N_Cycles_1-47.csv', header=0, parse_dates=[0], index_col=0, squeeze=True)

# data is an array with our already pre-processed dataset examples
kf = KFold(n_splits=3)
sum = 0
for train, test in kf.split(data):
    train_data = np.array(data)[train]
    test_data = np.array(data)[test]
    classifier = nltk.NaiveBayesClassifier.train(train_data)
    sum += nltk.classify.accuracy(classifier, test_data)
average = sum/3


{'fraction_non_alphanumeric': 0.13033175355450238, 'fraction_numerical': 0.026066350710900472, 'mean_word_length': 4.70945945945946, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6073701', 'n_tokens_mistral': 317, 'n_tokens_neox': 297, 'n_words': 72}