<jupyter_start><jupyter_text>Lasso for Causal BBN Structure Learning<jupyter_code>import pandas as pd
import json

df = pd.read_csv('../data/gmu-covid.csv')

with open('../data/gmu-covid.json', 'r') as f:
    meta = json.load(f)
df.shape<jupyter_output><empty_output><jupyter_text>Code<jupyter_code>from sklearn.linear_model import LogisticRegression
from itertools import combinations, chain
import operator
from functools import reduce
from typing import Tuple, Dict, List, Any

def get_n_way(X_cols: List[str], n_way=3):
    combs = (combinations(X_cols, n + 1) for n in range(n_way))
    combs = chain(*combs)
    combs = list(combs)
    return combs

def get_data(df_path: str, X_cols: List[str], y_col: str, n_way=3):
    def to_col_name(interaction):
        if len(interaction) == 1:
            return interaction[0]
        else:
            return '!'.join(interaction)
    
    def get_interaction(interaction):
        def multiply(r):
            vals = [r[col] for col in interaction]
            return reduce(operator.mul, vals, 1)
        
        return data.apply(multiply, axis=1)
        
    data = pd.read_csv(df_path)
    interactions = get_n_way(X_cols, n_way=n_way)
    
    d = {to_col_name(interaction): get_interaction(interaction) for interaction in interactions}
    d = {**d, **{y_col: data[y_col]}}
    
    df = pd.DataFrame(d)
    return df

def do_regression(X_cols: List[str], y_col: str, df: pd.DataFrame, solver='liblinear', penalty='l1', C=0.2) -> pd.DataFrame:
    X = df[X_cols]
    y = df[y_col]
    
    model = LogisticRegression(penalty=penalty, solver=solver, C=C)
    model.fit(X, y)
    
    return model

def extract_model_params(independent_cols: List[str], y_col: str, model: LogisticRegression):
    intercept = {'__intercept': model.intercept_[0]}
    indeps = {c: v for c, v in zip(independent_cols, model.coef_[0])}
    y = {'__dependent': y_col}
    
    d = {**y, **intercept}
    d = {**d, **indeps}
    
    return d

def to_robustness_indication(params: pd.DataFrame, ignore_neg_gt=-0.1, ignore_pos_lt=0.1):
    def is_robust(v):
        if v < ignore_neg_gt:
            return 0
        if v < ignore_pos_lt:
            return 0
        return 1

    return params[[c for c in params if c not in ['__intercept', '__dependent']]].applymap(is_robust)

def get_robust_stats(robust: pd.DataFrame, robust_threshold=0.9):
    s = robust.sum()
    p = s / robust.shape[0]
    i = s.index
    
    df = pd.DataFrame([{'name': name, 'count': count, 'percent': pct} for name, count, pct in zip(i, s, p)])
    df = df.sort_values(['count', 'percent', 'name'], ascending=[False, False, True])
    df = df[df['percent'] >= robust_threshold]
    return df
    
def do_robust_regression(X_cols: List[str], y_col: str, df_path: str, n_way=3, 
                         ignore_neg_gt=-0.1, ignore_pos_lt=0.1, 
                         n_regressions=10, solver='liblinear', penalty='l1', C=0.2,
                         robust_threshold=0.9):
    data = get_data(df_path, X_cols, y_col, n_way=n_way)
    frames = (data.sample(frac=0.9) for _ in range(n_regressions))
    
    independent_cols = [c for c in data.columns if c != y_col]
    models = (do_regression(independent_cols, y_col, data, solver=solver, penalty=penalty, C=C) for df in frames)
    
    params = pd.DataFrame((extract_model_params(independent_cols, y_col, m) for m in models))
    robust = to_robustness_indication(params, ignore_neg_gt, ignore_pos_lt)
    robust_stats = get_robust_stats(robust)
    
    return params, robust, robust_stats<jupyter_output><empty_output><jupyter_text>Ordering map<jupyter_code>ordering_map = {}

col_ordering = list(reversed(meta['ordering']))
for i, arr in enumerate(col_ordering):
    for col in arr:
        indeps = list(chain(*col_ordering[i+1:]))
        ordering_map[col] = indeps<jupyter_output><empty_output><jupyter_text>1. Fever ~ .<jupyter_code>','.join(ordering_map['Fever'])
df_path = '../data/gmu-covid.csv'
params, robust, robust_stats = do_robust_regression(ordering_map['Fever'], 'Fever', df_path)
params
robust
robust_stats<jupyter_output><empty_output><jupyter_text>2. Chestpain ~ .<jupyter_code>params, robust, robust_stats = do_robust_regression(ordering_map['Chestpain'], 'Chestpain', df_path)
params
robust
robust_stats<jupyter_output><empty_output><jupyter_text>2. Cough ~ .<jupyter_code>params, robust, robust_stats = do_robust_regression(ordering_map['Cough'], 'Cough', df_path)
robust_stats<jupyter_output><empty_output><jupyter_text>2. Difficultybreathing ~ .<jupyter_code>params, robust, robust_stats = do_robust_regression(ordering_map['Difficultybreathing'], 'Difficultybreathing', df_path)
robust_stats<jupyter_output><empty_output><jupyter_text>2. Sorethroat ~ .<jupyter_code>params, robust, robust_stats = do_robust_regression(ordering_map['Sorethroat'], 'Sorethroat', df_path)
robust_stats<jupyter_output><empty_output><jupyter_text>3. MuscleAches ~<jupyter_code>params, robust, robust_stats = do_robust_regression(ordering_map['MuscleAches'], 'MuscleAches', df_path)
robust_stats<jupyter_output><empty_output><jupyter_text>3. IgnoreNeuroSymp ~ .<jupyter_code>params, robust, robust_stats = do_robust_regression(ordering_map['IgnoreNeuroSymp'], 'IgnoreNeuroSymp', df_path)
robust_stats<jupyter_output><empty_output><jupyter_text>4. Fatigue ~ .<jupyter_code>params, robust, robust_stats = do_robust_regression(ordering_map['Fatigue'], 'Fatigue', df_path)
robust_stats<jupyter_output><empty_output><jupyter_text>5. Headaches ~ .<jupyter_code>params, robust, robust_stats = do_robust_regression(ordering_map['Headaches'], 'Headaches', df_path)
robust_stats<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12804232804232804, 'fraction_numerical': 0.008112874779541446, 'mean_word_length': 4.614851485148515, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '185220', 'license': 'Apache-2.0', 'n_tokens_mistral': 2136, 'n_tokens_neox': 2080, 'n_words': 395, 'path': 'jupyter/lasso-v2-01.ipynb', 'repo_name': 'oneoffcoder/lasso-bbn', 'size': '5670'}