<jupyter_start><jupyter_text>Initial EDA Goals:- ~~Filter data for only completed studies~~- Create NLP features- Filter feature list using RIG- Save processed data<jupyter_code>import pandas as pd
# import sklearn.tree
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
# from sklearn.feature_selection import mutual_info_classif
from sklearn.preprocessing import KBinsDiscretizer
# from sklearn.tree import DecisionTreeClassifier
# from sklearn import tree
from sklearn.pipeline import make_pipeline, Pipeline, FeatureUnion

from autotest_data.helper_functions import ModelTransformer, SampleExtractor, DenseTransformer<jupyter_output><empty_output><jupyter_text>Load the data saved from the database.<jupyter_code>df = pd.read_csv("../data/00raw/main_table.csv.gz")
kw_df = pd.read_csv("../data/00raw/keywords_table.csv.gz")<jupyter_output><empty_output><jupyter_text>It was not clear how to find completed studies from the database(there were two compelling flags)Checking the pivot table of those values to pick a filtering strategy.<jupyter_code>pd.crosstab(df['completion_date_type'], df['enrollment_type'])<jupyter_output><empty_output><jupyter_text>Complete the filter.<jupyter_code>df = df[df["completion_date_type"]=="Actual"]
kw_df = kw_df[kw_df["nct_id"].isin(df["nct_id"].unique())]<jupyter_output><empty_output><jupyter_text>Verify no missing data in the target column (enrollment)<jupyter_code>df.isna().sum()<jupyter_output><empty_output><jupyter_text>Investigate methods for selecting features by RIG- First create token counts / TF-IDF matrix from text column- Then try  using mutual_info_classifier    + not happy with the estimates process, even after discretizing the target- Then try computing with Decision Tree    + feature_importances are normamlized!    + digging into the tree give the optimal entropy change, still need to weight the average and divide<jupyter_code># X, Y = df["brief_title"], df[["enrollment"]]
# cvec = TfidfVectorizer(
#     stop_words="english",
#     max_features=50,
# )
#
# discritizer = KBinsDiscretizer(
#     n_bins=6,
#     encode="ordinal",
# )
# X_vec = cvec.fit_transform(X)
# Y_discrete = discritizer.fit_transform(Y)
# res = dict(zip(cvec.get_feature_names_out(),
#                mutual_info_classif(X_vec, Y_discrete[:,0], discrete_features=True)
#                ))
# dt = DecisionTreeClassifier(
#     criterion="entropy",
#     max_depth=1,
# )
#
# dt.fit(X_vec[:,[11]], Y_discrete[:,0])
# dt.tree_.node_count
# tree.plot_tree(dt)
# dt.tree_.impurity
# dt.tree_.capacity
# dt.tree_.n_node_samples<jupyter_output><empty_output><jupyter_text>create pipeline for vectorizing token counts<jupyter_code># create a pipeline to convert text columns to token count columns
# Using simple counts to be safe with no train/test split
binary = False
feats = 50

def text_pipeline(column_name, binary_flag, max_feats):
    return Pipeline([
                      ('text',SampleExtractor([column_name])),
                      ('dummify', CountVectorizer(binary=binary_flag,
                                                  max_features=max_feats,
                                                  stop_words="english")),
                      ('densify', DenseTransformer()),
                     ])

pipeline = Pipeline([
    ('features', FeatureUnion([
        ('brief_title', text_pipeline("brief_title", binary, feats)),
        ('official_title', text_pipeline("official_title", binary, feats)),
        ('description', text_pipeline("description", binary, feats)),
        ('cont_features', Pipeline([
                      ('continuous', SampleExtractor(['nct_id','enrollment'])), # potential bug when extracting single column
                      ])),
        ])),
])
# learn the vocabularies for each column
pipeline.fit(df)

# create column headers from vocabularies
col_names = []
col_names.extend(["brief_title_" + col for col in pipeline.steps[0][1].transformer_list[0][1].steps[1][1].get_feature_names_out()])
col_names.extend(["official_title_" + col for col in pipeline.steps[0][1].transformer_list[1][1].steps[1][1].get_feature_names_out()])
col_names.extend(["description_" + col for col in pipeline.steps[0][1].transformer_list[2][1].steps[1][1].get_feature_names_out()])
col_names.extend(["nct_id", "enrollment"])

# generate transformed data_frame
X = pd.DataFrame(pipeline.transform(df), columns=col_names)
X.shape
X.to_csv("../data/01interim/main_table_vectorized.csv.gz",index=False)
# naive tabulations.... too many keywords, switching to tokens
pd.crosstab(kw_df["nct_id"], kw_df["name"]).sum(axis=0).sort_values().tail(50)
cvec = CountVectorizer(
    stop_words="english",
    max_features=50,
    ngram_range=(1,3),
    binary=True,
)

cvec.fit(kw_df["name"])
cvec.get_feature_names_out()
X = pd.DataFrame(
    cvec.transform(kw_df["name"]).todense(),
    columns=cvec.get_feature_names_out()
)
X.sum()
# spot check count for random term
kw_df[kw_df["name"].str.lower().str.contains("syndrome")]
# remove index from kw_df version since filtered rows won't match X index
X["nct_id"] = kw_df["nct_id"].values

X = X.groupby("nct_id").sum()
X
# check number of nct_id values matches rows in X
kw_df["nct_id"].nunique()
# from deeper investigation when rows mismatched...
kw_df.groupby("nct_id").count()
X.to_csv("../data/01interim/keywords_table_vectorized.csv.gz",index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12633912079793128, 'fraction_numerical': 0.007942371629109716, 'mean_word_length': 4.76063829787234, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '259793', 'license': 'MIT', 'n_tokens_mistral': 1763, 'n_tokens_neox': 1732, 'n_words': 370, 'path': 'notebooks/eda.ipynb', 'repo_name': 'blomadam/autotest_data', 'size': '5414'}