<commit_before>#!/usr/bin/python

__author__ = 'ashwin'
__email__ = '<EMAIL>'

"""
This project tries to look at methods to unify knowledge bases with Machine
Learning Algorithms.

HYPOTHESIS: Can Knowledge base information (in this case the semantic web), improve
            the performance of Machine Learning Algorithms, given that the relation
            between variables is known?
"""

from lib.stdops.fileops import FileReader
import lib.test as test

# Preliminaries.
data_sep = ","
elim_var = ['$continuous$']
class_name = "Classify"

# Training and Test data files.
train_f_name = "dataset/adult_data/adult.data.temp"
test_f_name = "dataset/adult_data/adult.test.temp"

# File with column names and variable types.
var_f_name = "dataset/adult_data/adult.var"

# Create File Reader to read the data from a file and process it.
# This section the file reader only extracts the column values.
train_file_reader = FileReader(class_var_name=class_name)
train_file_reader.read_col_var_file(var_f_name, data_separator=data_sep)

# Get the data in the file.
train_file_reader.read_data(train_f_name, data_sep, elim_var)

# File Reader for test data set.
test_file_reader = FileReader(class_var_name=class_name)
test_file_reader.read_col_var_file(var_f_name, data_separator=data_sep)

# Get the data in the file.
test_file_reader.read_data(test_f_name, data_sep, elim_var)

# Test Naive Bayes.
score = test.test_naive_bayes(train_file_reader, test_file_reader)
print score

# Test Naive Bayes with cross product.
cross_prod_columns = [['education', 'marital-status'], ['sex', 'race']]
score = test.test_nb_cross_product(train_file_reader, test_file_reader, cross_prod_columns)
print score<commit_msg>Adding test cases<commit_after>#!/usr/bin/python

__author__ = 'ashwin'
__email__ = '<EMAIL>'

"""
This project tries to look at methods to unify knowledge bases with Machine
Learning Algorithms.

HYPOTHESIS: Can Knowledge base information (in this case the semantic web), improve
            the performance of Machine Learning Algorithms, given that the relation
            between variables is known?
"""

from lib.stdops.fileops import FileReader
import lib.test as test

# Preliminaries.
data_sep = ","
elim_var = ['$continuous$']
class_name = "Classify"

# Training and Test data files.
train_f_name = "dataset/adult_data/adult.data"
test_f_name = "dataset/adult_data/adult.test"

# File with column names and variable types.
var_f_name = "dataset/adult_data/adult.var"

# Create File Reader to read the data from a file and process it.
# This section the file reader only extracts the column values.
train_file_reader = FileReader(class_var_name=class_name)
train_file_reader.read_col_var_file(var_f_name, data_separator=data_sep)

# Get the data in the file.
train_file_reader.read_data(train_f_name, data_sep, elim_var)

# File Reader for test data set.
test_file_reader = FileReader(class_var_name=class_name)
test_file_reader.read_col_var_file(var_f_name, data_separator=data_sep)

# Get the data in the file.
test_file_reader.read_data(test_f_name, data_sep, elim_var)

# Test Naive Bayes.
score = test.test_naive_bayes(train_file_reader, test_file_reader)
print score

# Test Naive Bayes with cross product.
cross_prod_list = [[['workclass', 'education', 'marital-status']],
                   [['education', 'marital-status']],
                   [['marital-status', 'relationship']],
                   [['education', 'marital-status', 'relationship']]]

for cross_prod_columns in cross_prod_list:
    score = test.test_nb_cross_product(train_file_reader, test_file_reader, cross_prod_columns)
    print cross_prod_columns
    print score
{'fraction_non_alphanumeric': 0.08750341810226962, 'fraction_numerical': 0.0, 'mean_word_length': 5.1375838926174495, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2023463', 'n_tokens_mistral': 1209, 'n_tokens_neox': 1159, 'n_words': 329}