<commit_before>import unittest, random, sys, time
sys.path.extend(['.','..','py'])
import h2o, h2o_browse as h2b, h2o_exec as h2e, h2o_hosts, h2o_import as h2i

# new ...ability to reference cols
# src[ src$age<17 && src$zip=95120 && ... , ]
# can specify values for enums ..values are 0 thru n-1 for n enums
print "FIX!: need to test the && and || reduction operators"
initList = [
        # ('r.1', 'r.1=c(1.3,0,1,2,3,4,5)'),
        # ('r2.hex', 'r2.hex=c(2.3,0,1,2,3,4,5)'),
        # ('r3.hex', 'r3.hex=c(4.3,0,1,2,3,4,5)'),
        ('r.1', 'r.1=i.hex'),
        ('r.1', 'r.1=i.hex'),
        ('r2.hex', 'r2.hex=i.hex'),
        ('r3.hex', 'r3.hex=i.hex'),
        # not supported. should? also row vector assign?
        # ('r.1',  'r.1[1,]=3.3'),

        # ('x', 'x=r.1[,1]; rcnt=nrow(x)-sum(is.na(x))'),
        # ('x', 'x=r.1[,1]; total=sum(ifelse(is.na(x),0,x)); rcnt=nrow(x)-sum(is.na(x))'),
        # ('x', 'x=r.1[,1]; total=sum(ifelse(is.na(x),0,x)); rcnt=nrow(x)-sum(is.na(x)); mean=total / rcnt'),
        # ('x', 'x=r.1[,1]; total=sum(ifelse(is.na(x),0,x)); rcnt=nrow(x)-sum(is.na(x)); mean=total / rcnt; x=ifelse(is.na(x),mean,x)'),
        ]

# apply: return vector or array or list of values..applying function to margins of array or matrix
# margins: either rows(1), columns(2) or both(1:2)
# "apply(r.1,2,function(x){total=sum(ifelse(is.na(x),0,x)); rcnt=nrow(x)-sum(is.na(x)); mean=0.0; ifelse(is.na(x),mean,x)})",
# doesn't work. Should work according to earl
# 'r.1[is.na(r.1)]<-0',
# works
# 'r.1=apply(r.1,2,function(x){ifelse(is.na(x),0,x)})',
# "mean=function(x){apply(x,2,sum)/nrow(x)};mean(r.1)",

deepIfElse = ""
closeParen = 0
for i in range(20):
    # two, so we complement
    deepIfElse += "ifelse(1,0,"
    deepIfElse += "ifelse(0,1,"
    closeParen += 1
# now add all the closeParens you need
deepIfElse += "0"
for i in range(closeParen):
    deepIfElse += '))'

exprListFull = [
    deepIfElse,
    # this is supposed to work on vendavo branch?
    # "apply(r.1, 1, function(x) { x+1 })",
    # fails if row based ..okay if col base
    # "apply(r.1, 1, function(x) { mean(x) })",
    # "apply(r.1, 1, function(x) { sd(x) })",
    # "apply(r.1, 1, function(x) { sd(x)/mean(x) })",

    "apply(r.1, 2, function(x) { mean(x) })",
    "apply(r.1, 2, function(x) { sd(x) })",
    "apply(r.1, 2, function(x) { sd(x)/mean(x) })",

    "ddply(r.1,c(3),nrow)",
    # More complex multi-return
    "ddply(r.1,c(3),function(x) {c(mean(x[,2]),mean(x[,3]))})",
    "ddply(r.1,c(7),nrow)",
<commit_msg>restrict ddply<commit_after>import unittest, random, sys, time
sys.path.extend(['.','..','py'])
import h2o, h2o_browse as h2b, h2o_exec as h2e, h2o_hosts, h2o_import as h2i

# new ...ability to reference cols
# src[ src$age<17 && src$zip=95120 && ... , ]
# can specify values for enums ..values are 0 thru n-1 for n enums
print "FIX!: need to test the && and || reduction operators"
initList = [
        # ('r.1', 'r.1=c(1.3,0,1,2,3,4,5)'),
        # ('r2.hex', 'r2.hex=c(2.3,0,1,2,3,4,5)'),
        # ('r3.hex', 'r3.hex=c(4.3,0,1,2,3,4,5)'),
        ('r.1', 'r.1=i.hex'),
        ('r.1', 'r.1=i.hex'),
        ('r2.hex', 'r2.hex=i.hex'),
        ('r3.hex', 'r3.hex=i.hex'),
        # not supported. should? also row vector assign?
        # ('r.1',  'r.1[1,]=3.3'),

        # ('x', 'x=r.1[,1]; rcnt=nrow(x)-sum(is.na(x))'),
        # ('x', 'x=r.1[,1]; total=sum(ifelse(is.na(x),0,x)); rcnt=nrow(x)-sum(is.na(x))'),
        # ('x', 'x=r.1[,1]; total=sum(ifelse(is.na(x),0,x)); rcnt=nrow(x)-sum(is.na(x)); mean=total / rcnt'),
        # ('x', 'x=r.1[,1]; total=sum(ifelse(is.na(x),0,x)); rcnt=nrow(x)-sum(is.na(x)); mean=total / rcnt; x=ifelse(is.na(x),mean,x)'),
        ]

# apply: return vector or array or list of values..applying function to margins of array or matrix
# margins: either rows(1), columns(2) or both(1:2)
# "apply(r.1,2,function(x){total=sum(ifelse(is.na(x),0,x)); rcnt=nrow(x)-sum(is.na(x)); mean=0.0; ifelse(is.na(x),mean,x)})",
# doesn't work. Should work according to earl
# 'r.1[is.na(r.1)]<-0',
# works
# 'r.1=apply(r.1,2,function(x){ifelse(is.na(x),0,x)})',
# "mean=function(x){apply(x,2,sum)/nrow(x)};mean(r.1)",

deepIfElse = ""
closeParen = 0
for i in range(20):
    # two, so we complement
    deepIfElse += "ifelse(1,0,"
    deepIfElse += "ifelse(0,1,"
    closeParen += 1
# now add all the closeParens you need
deepIfElse += "0"
for i in range(closeParen):
    deepIfElse += '))'

exprListFull = [
    deepIfElse,
    # this is supposed to work on vendavo branch?
    # "apply(r.1, 1, function(x) { x+1 })",
    # fails if row based ..okay if col base
    # "apply(r.1, 1, function(x) { mean(x) })",
    # "apply(r.1, 1, function(x) { sd(x) })",
    # "apply(r.1, 1, function(x) { sd(x)/mean(x) })",

    "apply(r.1, 2, function(x) { mean(x) })",
    "apply(r.1, 2, function(x) { sd(x) })",
    "apply(r.1, 2, function(x) { sd(x)/mean(x) })",

    "ddply(r.1,c(3),nrow)",
    # More complex multi-return
    # ddply can only return one thing
    # "ddply(r.1,c(3),function(x) {c(mean(x[,2]),mean(x[,3]))})",
    "ddply(r.1,c(7),nrow)",

{'fraction_non_alphanumeric': 0.2577987051206592, 'fraction_numerical': 0.04747890916225231, 'mean_word_length': 3.4562937062937062, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5707517', 'n_tokens_mistral': 2387, 'n_tokens_neox': 2306, 'n_words': 433}