<jupyter_start><jupyter_text>Feature Importance VerificationIn this notebook, I will be running a comparitive analysis to try to disprove the following hypothesis:If the current implementation of permutation selection for deep learning is correct, training a model without the same features should result in the loss being what was found in the original table.I will be using the ADULTs dataset first, as it is simpler to set up and grade, and then Rossman.The data will be split into 70% train, 20% validation, and 10% test randomly from `train_test_split` in the sklearn library.Each test will be done with five epochs.  Libraries<jupyter_code>from fastai import *
from fastai.tabular import *
from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>path = untar_data(URLs.ADULT_SAMPLE)
df = pd.read_csv(path/'adult.csv')
random_seed(5)
train, test = train_test_split(df, test_size = 0.1)
len(train), len(test)
dep_var = 'salary'
cats = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']
conts = ['age', 'fnlwgt', 'education-num']
procs = [FillMissing, Categorify, Normalize]
data = (TabularList.from_df(train, path=path, cat_names=cats, cont_names=conts, 
                            procs=procs)
                           .split_by_rand_pct(0.2)
                           .label_from_df(cols=dep_var)
                           .databunch())
data_test = (TabularList.from_df(test, path=path, cat_names=cats, cont_names=conts, 
                            procs=procs, processor=data.processor)
                           .split_none()
                           .label_from_df(cols=dep_var)
                           .databunch())<jupyter_output><empty_output><jupyter_text>Baseline<jupyter_code>learn.data.valid_dl = data.valid_dl
learn = tabular_learner(data, layers=[200,100], metrics=accuracy)
learn.fit_one_cycle(5, 1e-2)<jupyter_output><empty_output><jupyter_text>Feature ImportanceHere I will be using the permutation-based feature importance algorithm on the test set to derive the hypothetical overall importance<jupyter_code>cats, conts
  cats = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']
  conts = ['age', 'fnlwgt', 'education-num']
def tfeature_importance(learn:Learner, cats:list, conts:list, dep_var:str, test:DataFrame):
  data = learn.data.train_ds.x
  procs = data.procs
  dt = (TabularList.from_df(test, path='', cat_names=cats, cont_names=conts, 
                            procs=procs)
                           .split_none()
                           .label_from_df(cols=dep_var)
                           .databunch())
  learn.data.valid_dl = dt.train_dl
  loss0 = float(learn.validate()[1])
  cats = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']
  conts = ['age', 'fnlwgt', 'education-num']
  types = [cats, conts]
  fi=dict()
  for j, t in enumerate(types):
    for i, c in enumerate(t):
      print(c)
      base = test.copy()
      base[c] = base[c].sample(n=len(base), replace=True).reset_index(drop=True)
      cats = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']
      conts = ['age', 'fnlwgt', 'education-num']
      dt = (TabularList.from_df(base, path='', cat_names=cats, cont_names=conts, 
                            procs=procs)
                           .split_none()
                           .label_from_df(cols=dep_var)
                           .databunch())
      learn.data.valid_dl = dt.train_dl
      fi[c] = float(learn.validate()[1]) - loss0
      
  d = sorted(fi.items(), key =lambda kv: kv[1], reverse=True)
  df = pd.DataFrame({'Variable': [l for l, v in d], 'Accuracy': [v for l, v in d]})
  df['Type'] = ''
  for x in range(len(df)):
    if df['Variable'].iloc[x] in cats:
      df['Type'].iloc[x] = 'categorical'
    if df['Variable'].iloc[x] in conts:
      df['Type'].iloc[x] = 'continuous'
  return df                  
res = tfeature_importance(learn, cats, conts, dep_var, test)<jupyter_output>/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:190: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self._setitem_with_indexer(indexer, value)<jupyter_text>So here, hypothetically what we should see is a boost in performance when we drop everything that is positive.Next I will get the actual ground truth level for the test set as well, to compare the baselines<jupyter_code>learn.data.valid_dl = data_test.train_dl
learn.validate()<jupyter_output><empty_output><jupyter_text>**83.91%** is the baseline we will be going with  Dropping ColumnsHere I make a for loop that drops a variable from the list, continues to the next and trains for 5, gets our test result, and continues on  Functions<jupyter_code>def random_seed(seed_value):
    import random 
    random.seed(seed_value) # Python
    import numpy as np
    np.random.seed(seed_value) # cpu vars
    import torch
    torch.manual_seed(seed_value) # cpu  vars
    
    if torch.cuda.is_available(): 
        torch.cuda.manual_seed(seed_value)
        torch.cuda.manual_seed_all(seed_value) # gpu vars
        torch.backends.cudnn.deterministic = True  #needed
        torch.backends.cudnn.benchmark = False
class DropTest:
  def __init__(self, cat_vars:list, cont_vars:list, dep_var:str, df:DataFrame):
    self.cats = cat_vars
    self.conts = cont_vars
    self.dep = dep_var
    self.df = df
    self.procs = [FillMissing, Categorify, Normalize]
    self.res = pd.DataFrame(columns=['Variable', 'Accuracy'])
    self.types = [self.cats, self.conts]
    
  def calc_drop(self):
    train, test = train_test_split(self.df, test_size=0.1)
    k = 0
    for j, t in enumerate(self.types):
      for i, c in enumerate(t):
        random_seed(5)
        cat_copy = self.cats.copy()
        cont_copy = self.conts.copy()
        if c in cat_copy:
          cat_copy.remove(c)
        else:
          cont_copy.remove(c)
        
        df = self.df.drop(c, axis=1)
        
        data = (TabularList.from_df(train, path='', cat_names=cat_copy,
                                   cont_names=cont_copy, procs=self.procs)
               .split_by_rand_pct(0.2)
               .label_from_df(cols=self.dep)
               .databunch())
        data_test = (TabularList.from_df(test, path='', cat_names=cat_copy,
                                        cont_names=cont_copy, procs=self.procs,
                                        processor=data.processor)
                     .split_none()
                     .label_from_df(cols=self.dep)
                     .databunch())
        learn = tabular_learner(data, layers=[200,100], metrics=accuracy)
        learn.fit_one_cycle(5, 1e-2)
        learn.data.valid_dl = data_test.train_dl
        val = learn.validate()
        val = float(val[1])
        self.res.loc[k] = [str(c), val]
        k += 1<jupyter_output><empty_output><jupyter_text>DropTest<jupyter_code>t = DropTest(cats, conts, 'salary', df)
t.calc_drop()
re= t.res
re.sort_values('Accuracy',ascending=False)
res.sort_values('Accuracy', ascending=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11620585819874076, 'fraction_numerical': 0.007254311524774158, 'mean_word_length': 2.928494623655914, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '453678', 'license': 'Apache-2.0', 'n_tokens_mistral': 2449, 'n_tokens_neox': 2354, 'n_words': 552, 'path': 'Feature Importance/FeatureImportanceExperimentation.ipynb', 'repo_name': 'muellerzr/FastAI-Experiments-and-tips', 'size': '7306'}