<commit_before>using DataFrames
using DecisionTree
using StatsBase

cd("/Users/pstey/projects_code/julia_tutorials/statistics/code")

ckd = readtable("../data/chronic_kidney_disease.csv")

function hotdeck(x)
    non_na = [!isna(z) for z in x]
    vals = x[non_na]
    shuffle!(vals)
    xnew = similar(x)
    n = length(x)
    for i = 1:n
        xnew[i] = isna(x[i]) ? rand(vals) : x[i]
    end
    return xnew
end


cols = [:age,
        :blood_pressure,
        :specific_gravity,
        :albumin,
        :sugar,
        :blood_urea,
        :serum_creatinine,
        :sodium,
        :potassium,
        :hemoglobin,
        :packed_cell_volume,
        :white_blood_cell_count,
        :red_blood_cell_count,
        :class]


ckd_subset = ckd[:, cols]

function hotdeck_dataframe!(dat)
    p = ncol(dat)
    for j = 1:p
        dat[:, j] = hotdeck(dat[:, j])
    end
end

hotdeck_dataframe!(ckd_subset)



## Fitting Random Forest
# Cast data to numeric arrays
labels = convert(Array{Int,1}, map(x -> x == "ckd" ? 1 : 0, ckd_subset[:class]))
features = convert(Array, ckd_subset[:, 1:(end - 1)])

<commit_msg>fix parameter to force conversion to array of floats<commit_after>using DataFrames
using DecisionTree
using StatsBase

cd("/Users/pstey/projects_code/julia_tutorials/statistics/code")

ckd = readtable("../data/chronic_kidney_disease.csv")

function hotdeck(x)
    non_na = [!isna(z) for z in x]
    vals = x[non_na]
    shuffle!(vals)
    xnew = similar(x)
    n = length(x)
    for i = 1:n
        xnew[i] = isna(x[i]) ? rand(vals) : x[i]
    end
    return xnew
end


cols = [:age,
        :blood_pressure,
        :specific_gravity,
        :albumin,
        :sugar,
        :blood_urea,
        :serum_creatinine,
        :sodium,
        :potassium,
        :hemoglobin,
        :packed_cell_volume,
        :white_blood_cell_count,
        :red_blood_cell_count,
        :class]


ckd_subset = ckd[:, cols]

function hotdeck_dataframe!(dat)
    p = ncol(dat)
    for j = 1:p
        dat[:, j] = hotdeck(dat[:, j])
    end
end

hotdeck_dataframe!(ckd_subset)



## Fitting Random Forest
# Cast data to numeric arrays
labels = convert(Array{Int,1}, map(x -> x == "ckd" ? 1 : 0, ckd_subset[:class]))
features = convert(Array{Float64, 2}, ckd_subset[:, 1:(end - 1)])


{'fraction_non_alphanumeric': 0.13324595893403232, 'fraction_numerical': 0.007426823940585408, 'mean_word_length': 2.741830065359477, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4368623', 'n_tokens_mistral': 919, 'n_tokens_neox': 860, 'n_words': 185}