<commit_before>REGISTER 'naive-bayes.r' using com.cwoodson.pigaddons.rpig.RScriptEngine as rfuncs;

test_data = LOAD 'naive-bayes.data.test' AS (test_result:int, a:int, b:int, c:int, d:int, e:int, f:int, g:int, h:int, i:int, j:int, k:int, l:int, m:int, n:int, o:int, p:int);

test_spam = FILTER test_data BY test_result == 1;
test_spam1 = GROUP test_spam ALL;
test_nospam = FILTER test_data BY test_result == 0;
test_nospam1 = GROUP test_nospam ALL;

test_spam_cnt = FOREACH test_spam1 GENERATE SUM(test_spam.a), SUM(test_spam.b), SUM(test_spam.c), SUM(test_spam.d), SUM(test_spam.e), SUM(test_spam.f) , SUM(test_spam.g), SUM(test_spam.h), SUM(test_spam.i), SUM(test_spam.j), SUM(test_spam.k), SUM(test_spam.l), SUM(test_spam.m), SUM(test_spam.n), SUM(test_spam.o), SUM(test_spam.p), COUNT(test_spam);
test_nospam_cnt = FOREACH test_nospam1 GENERATE SUM(test_nospam.a), SUM(test_nospam.b), SUM(test_nospam.c), SUM(test_nospam.d), SUM(test_nospam.e), SUM(test_nospam.f), SUM(test_nospam.g), SUM(test_nospam.h), SUM(test_nospam.i), SUM(test_nospam.j), SUM(test_nospam.k), SUM(test_nospam.l), SUM(test_nospam.m), SUM(test_nospam.n), SUM(test_nospam.o), SUM(test_nospam.p), COUNT(test_nospam);

real_data = LOAD 'naive-bayes.data' AS (a:int, b:int, c:int, d:int, e:int, f:int, g:int, h:int, i:int, j:int, k:int, l:int, m:int, n:int, o:int, p:int);

output = FOREACH real_data { fields = TOTUPLE($0 ..); GENERATE rfuncs.CalcProb(fields, test_spam_cnt, test_nospam_cnt) as prob:(spam:int, pnotspam:double, pisspam:double); }

STORE output INTO 'output/naive-bayes';<commit_msg>Bug Fixing<commit_after>REGISTER 'naive-bayes.r' using com.cwoodson.pigaddons.rpig.RScriptEngine as rfuncs;

test_data = LOAD 'naive-bayes.data.test' AS (test_result:int, a:int, b:int, c:int, d:int, e:int, f:int, g:int, h:int, i:int, j:int, k:int, l:int, m:int, n:int, o:int, p:int);

test_spam = FILTER test_data BY test_result == 1;
test_spam1 = GROUP test_spam ALL;
test_nospam = FILTER test_data BY test_result == 0;
test_nospam1 = GROUP test_nospam ALL;

test_spam_cnt = FOREACH test_spam1 GENERATE TOTUPLE(SUM(test_spam.a), SUM(test_spam.b), SUM(test_spam.c), SUM(test_spam.d), SUM(test_spam.e), SUM(test_spam.f) , SUM(test_spam.g), SUM(test_spam.h), SUM(test_spam.i), SUM(test_spam.j), SUM(test_spam.k), SUM(test_spam.l), SUM(test_spam.m), SUM(test_spam.n), SUM(test_spam.o), SUM(test_spam.p), COUNT(test_spam)) AS spam_test_data, 'xx' AS join_key;
test_nospam_cnt = FOREACH test_nospam1 GENERATE TOTUPLE(SUM(test_nospam.a), SUM(test_nospam.b), SUM(test_nospam.c), SUM(test_nospam.d), SUM(test_nospam.e), SUM(test_nospam.f), SUM(test_nospam.g), SUM(test_nospam.h), SUM(test_nospam.i), SUM(test_nospam.j), SUM(test_nospam.k), SUM(test_nospam.l), SUM(test_nospam.m), SUM(test_nospam.n), SUM(test_nospam.o), SUM(test_nospam.p), COUNT(test_nospam)) AS notspam_test_data, 'xx' AS join_key;

real_data = LOAD 'naive-bayes.data' AS (fields:(a:int, b:int, c:int, d:int, e:int, f:int, g:int, h:int, i:int, j:int, k:int, l:int, m:int, n:int, o:int, p:int));

real_data_aux = FOREACH real_data GENERATE fields, 'xx' AS join_key;

real_data_joined = JOIN real_data_aux BY join_key LEFT, test_spam_cnt BY join_key;
real_data_joined2 = JOIN real_data_joined BY join_key LEFT, test_notspam_cnt BY join_key;

describe real_data_joined2;

--result = FOREACH real_data GENERATE rfuncs.CalcProb(fields, spam_test_data, notspam_test_data) as prob:(spam:int, pnotspam:double, pisspam:double);

--STORE result INTO 'output/naive-bayes';
{'fraction_non_alphanumeric': 0.16036655211912945, 'fraction_numerical': 0.00429553264604811, 'mean_word_length': 7.085648148148148, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2849265', 'n_tokens_mistral': 1709, 'n_tokens_neox': 1594, 'n_words': 288}