<jupyter_start><jupyter_text>Smoking prevelance- Parameter variation `VerificationSmoking` in Anylogic<jupyter_code>library(data.table)
library(xtable)
source("../src/utils.R")
# read data files
m = fread("../output/data/mortality-verification-smoking.csv")
p = fread("../output/data/parameters-verification-smoking.csv")
e = fread("../output/data/environment-verification-smoking.csv")
# select parameters of interest
parameters = c("smoking_rank_slope_exp_coeff")
setorderv(p, parameters)

# redefine iteration and replicate indexes
p[, niteration := .GRP, by = parameters]
p[, nreplicate := 1:.N, by = niteration]
np = p[, c("iteration", "replicate", "niteration", "nreplicate", parameters), with = FALSE]
# environment data

# create columns from arrays
vars = paste0("income", 1:5)
e = extractColumns(e, "prop_income_group", vars)
vars = paste0("le", 1:5)
e = extractColumns(e, "le_income_group",  vars)
vars = paste0("smoking",  1:5)
e = extractColumns(e, "smoking_income_group",  vars)

# merge files
e = merge(e, np, by = c("iteration", "replicate"))
setorderv(e, parameters)

# create tables with distributions
nhis = readRDS("../output/data/smoking_dist_nhis2019.rds")
s = melt(e, id.vars = c("niteration", "nreplicate"), measure.vars = patterns("^smoking[0-9]"), 
    value.name = "smoking_prop", variable.name = "income_group")

print(paste("Population no-treatment", round(mean(e[niteration == 1, population]), 0)))
print(paste("Population treatment", round(mean(e[niteration == 2, population]), 0)))

cf = s[niteration == 1, .(smoking_prop_cf = mean(smoking_prop)), income_group]
cf[, income_group := as.numeric(gsub("[a-z]+", "", income_group))]
cf = rbind(cf, data.table(income_group = 9, smoking_prop_cf  = mean(e[niteration == 1, smokers])))

trt = s[niteration == 2, .(smoking_prop_trt = mean(smoking_prop)), income_group]
trt[, income_group := as.numeric(gsub("[a-z]+", "", income_group))]
trt = rbind(trt, data.table(income_group = 9, smoking_prop_trt  = mean(e[niteration == 2, smokers])))

tab = Reduce(function(...) merge(..., all = TRUE, by = "income_group"), list(nhis, cf, trt))
setnames(tab, names(tab), 
    c("Income quintile" , "NHIS 2019", "MIA non-treatment", "MIA treatment"))

# latex table 
print(xtable(tab,
    caption = "Proportion smoking by income quintile"),
    table.placement = "htp",
    caption.placement = "top",
    include.rownames  = FALSE
)
# le by smoking 
l = m[age > 40, .(le = mean(age)), .(smoker, income_group)]
setorder(l, income_group, smoker)
sl = m[age > 40, .(le = mean(age)), income_group]
setorder(sl, income_group)
sl
m = merge(m, np, by = c("iteration", "replicate"))

m[, .(le = mean(age)), niteration]
m[age > 50, .(le = mean(age)), niteration]
m[age < 30, .(le = mean(age)), niteration]


diff(m[, .(le = mean(age)), .(smoker)]$le[c(2, 1)])

# exploring some values
a = mean(e[niteration == 1, le5])
b = mean(e[niteration == 2, le5])
a - b

a = mean(e[niteration == 1, le1])
b = mean(e[niteration == 2, le1])
a - b<jupyter_output><empty_output><jupyter_text>Income measures<jupyter_code>inc = fread("../models/MobHealthRecycling/output/testing/income-1-1.csv")
inc[, kid_income := gsub("\\[|\\]", "", kid_income)]
inc[, parent_income := gsub("\\[|\\]", "", parent_income)]
sinc = inc[, .(kid = as.numeric(unlist(strsplit(kid_income, ','))), 
    parent = as.numeric(unlist(strsplit(parent_income, ',')))), 
    by = .(model_time, county)]
sinc = merge(sinc, 
    inc[, .(model_time, county, rank_slope, rank_absolute, rank_correlation, cohort_size )], 
    by = c("county", "model_time"))
head(sinc)
sinc[, kid_rank := perc.rank(kid), .(model_time)]
sinc[, parent_rank := perc.rank(parent), .(model_time)]

reg = function(kid_income, parent_income, relative = TRUE) {
    m = lm(kid_income ~ parent_income)
    c = coef(m)
    if (relative) { return(c[2])}
    else {
        return (c[1]  + 0.25 * c[2])
    }
}

sinc[, spearman := cor(kid, parent, method = "spearman"), model_time]
sinc[, cor_rank := cor(kid_rank, parent_rank), model_time]
sinc[, im := reg(kid_rank, parent_rank, TRUE), model_time]
sinc[, am := reg(kid_rank, parent_rank, FALSE), model_time]
sinc[, order := 1:.N, model_time]
sinc = sinc[order == 1]

summary(sinc$rank_slope - sinc$im)
summary(sinc$rank_absolute - sinc$am)
summary(sinc$rank_correlation - sinc$spearman)
summary(sinc$rank_correlation - sinc$cor_rank)
cor(sinc$rank_slope, sinc$im)
cor(sinc$rank_absolute, sinc$am)
cor(sinc$rank_correlation, sinc$spearman)
cor(sinc$rank_correlation, sinc$cor_rank)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18504631671812968, 'fraction_numerical': 0.012792236435818262, 'mean_word_length': 5.525179856115108, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '156854', 'license': 'MIT', 'n_tokens_mistral': 1815, 'n_tokens_neox': 1695, 'n_words': 362, 'path': 'notebooks/verification-smoking.ipynb', 'repo_name': 'sdaza/income-mobility-mortality-abm', 'size': '4534'}