<jupyter_start><jupyter_text>transforms.tabular> Contains all the transforms relevent to deployment in the `fastai` tabular library Check to make sure tabular is installed, if not throw error:<jupyter_code>#export
from fastinference_pytorch.soft_dependencies import SoftDependencies
if not SoftDependencies.check()['tab']:
    raise ImportError("The tabular module is not installed.")
#hide
from nbdev.showdoc import *
#export
import numpy as np
import torch
from torch import tensor
from fastcore.utils import store_attr
from fastinference_pytorch.rebuild import load_data
data = load_data('../')
data.keys()<jupyter_output><empty_output><jupyter_text>For an example we'll use the first five rows of the `ADULT_SAMPLE` dataset, which I have converted to a `NumPy` array below:<jupyter_code>#export
class Encoder():
    """
    Single class which handles tabular pre-processing. Will extract
    all relevent information from `dictionary` needed for transformations
    
    Arguments:
    `dictionary`: dict, export from `fastinference`
    """
    
    can_decode,order = True, 1
    def __init__(self, dictionary):
        self.fm = dictionary['FillMissing']
        self.categorify = dictionary['Categorify']
        self.norm = dictionary['Normalize']
        self.encoder = dictionary['Encoder']
        for var in self.categorify['classes']:
            self.categorify['classes'][var][np.nan] = 0
        self.tensorize = Tensorize(self.encoder)
        
    def __call__(self, x, decode=False):
        if not decode:
            x = self._fill_missing(x)
            x = self._categorify(x)
            x = self._normalize(x)
            x = self.tensorize(x)
            return x
    
    def _fill_missing(self, x):
        "Fills in mising data in `conts` and potentially generates a new categorical column"
        for idx, name in self.encoder['conts'].items():
            if name in self.fm['na_dict'].keys():
                nan = np.argwhere(x[:,idx]!=x[:,idx])
                x[:,idx][nan] = self.fm['na_dict'][name]
            if self.fm['add_col']:
                x = np.append(x, np.expand_dims(x[:,idx]==x[:,idx],1), 1)
        return x
    
    def _categorify(self, x):
        "Encodes categorical data in `x` based on `self.categorify"
        for idx, name in self.encoder['cats'].items():
            x[:,idx] = [self.categorify['classes'][name][i] for i in x[:,idx]]
        return x
    
    def _normalize(self, x):
        "Normalize continous data in `x` based on `self.normalize`"
        for idx, name in self.encoder['conts'].items():
            x[:,idx] = (x[:,idx]-self.norm['means'][name])/self.norm['stds'][name]
        return x
#export
class Tensorize():
    def __init__(self, enc:Encoder):
        """
        Converts numpy array to a `tensor`.
        
        Params:
        
        `enc`: Encoder exported from `fastinference`
        """
        cat_idxs = list(enc['cats'].keys())
        cont_idxs = list(enc['conts'].keys())
        store_attr(self, 'cat_idxs, cont_idxs')
    
    def __call__(self, x):
        cat = np.take(x, self.cat_idxs, axis=1).astype('int')
        cont = np.take(x, self.cont_idxs, axis=1).astype('float')
        return tensor(cat), tensor(cont)
#slow
import pandas as pd
df = pd.read_csv('/home/ml1/.fastai/data/adult_sample/adult.csv').to_numpy()
bs = 512
enc = Encoder(data)
t_df = enc(df)
#export
class NumpyDataset():
    def __init__(self, cats, conts,bs):
        "A simply dataset for NumPy after grouping"
        store_attr(self, 'cats,conts,bs')
        self.n_batches = len(cats) // self.bs + (0 if len(cats)%self.bs == 0 else 1)
    def __getitem__(self, idx): return (self.cats[idx:idx+self.bs], self.conts[idx:idx+self.bs])
    
    def __len__(self): return self.n_batches
from torch.utils.data import DataLoader
dset = NumpyDataset(*t_df, 128)
dl = DataLoader(dset, batch_size=1)
o = next(iter(dl))
o
for batch in dset:
    print(batch[0][0])
t = Tensorize(enc)
t(row)
%%time
_ = np.take(arr, list(enc.encoder['cats'].keys())).astype('int')
enc.encoder['cats'].keys()
%%timeit
_ = np.append(arr, np.expand_dims(arr[:,4]==arr[:,4],1), 1)
%%timeit
b = np.zeros((arr.shape[0], arr.shape[1]+1), dtype=np.object)
b[:len(arr), :-1] = arr
#export
def FillMissing(arr, procs):
    "Fills in missing data in `conts` and potentially generates a new categorical column"
    for idx, name in procs['Inputs']['conts'].items():
        if name in procs['FillMissing']['na_dict'].keys():
            nan = np.argwhere(arr[:,idx]!=arr[:,idx])
            arr[:,idx][nan] = procs['FillMissing']['na_dict'][name]
        if procs['FillMissing']['add_col']:
            arr = np.append(arr, np.expand_dims(arr[:,4]==arr[:,4],1), 1)
    return arr
show_doc(FillMissing)<jupyter_output><empty_output><jupyter_text>`arr` is expected to be a `NumPy` array, while `procs` should be the pre-processing dictionary exported after training<jupyter_code>#slow
df[0]
#slow
df = FillMissing(df, procs)
#slow
df[0]<jupyter_output><empty_output><jupyter_text>Three `bool` columns were added at the end for our potential missing numerical values (if `True` they exist)<jupyter_code>#export
def Categorize(arr, procs):
    "Encodes categorical data in `arr` based on `procs`"
    for idx, name in procs['Inputs']['cats'].items():
        arr[:,idx] = [procs['Categorize'][name][i] for i in arr[:,idx]]
    return arr
show_doc(Categorize)<jupyter_output><empty_output><jupyter_text>`arr` is expected to be a `NumPy` array, while `procs` should be the pre-processing dictionary exported after training<jupyter_code>#slow
df[0]
#slow
df = Categorize(df, procs)
#slow
df[0]<jupyter_output><empty_output><jupyter_text>Our categorical variables are now all converted to integers. Any left as strings are not used by the model and are ignored at inference time.<jupyter_code>#export
def Normalize(arr, procs):
    "Normalizes continous data in `arr` based on `procs`"
    for idx, name in procs['Inputs']['conts'].items():
        arr[:,idx] = (arr[:,idx]-procs['Normalize'][name]['mean'])/procs['Normalize'][name]['std']
        return arr
show_doc(Normalize)<jupyter_output><empty_output><jupyter_text>`arr` is expected to be a `NumPy` array, while `procs` should be the pre-processing dictionary exported after training<jupyter_code>#slow
df[0]
#slow
df = Normalize(df, procs)
#slow
df[0]<jupyter_output><empty_output><jupyter_text>Our continous variables have now been adjusted for the model<jupyter_code>#export
def apply_procs(arr, procs):
    "Apply test-time pre-processing on `NumPy` array input"
    arr = FillMissing(arr, procs)
    arr = Categorize(arr, procs)
    arr = Normalize(arr, procs)
    return arr
show_doc(apply_procs)<jupyter_output><empty_output><jupyter_text>The specific order in which the pre-processing is done must occur, as `Categorify` can increase by a few columns from `FillMissing` if multiple `is_na` columns are added<jupyter_code>#slow
df = pd.read_csv('/home/ml1/.fastai/data/adult_sample/adult.csv')
df = df.head().to_numpy()
#slow
df[0]
#slow
df = apply_procs(df, procs)
#slow
df[0]
#export
class TabularDataset():
    "A tabular `PyTorch` dataset based on `procs` with batch size `bs` on `device`"
    def __init__(self, arr, procs, bs=64, device='cuda'):
        "Stores array, grabs the indicies for `cats` and `conts`, and generates batches"
        self.arr = arr
        self.cat_idxs = procs['Inputs']['cats'].keys()
        self.cont_idxs = procs['Inputs']['conts'].keys()
        self.bs = bs
        self.device = device
        self.make_batches()
        
    def __getitem__(self, x):
        "Grabs one batch of data and converts it to the proper type"
        row = [self.batches[x][:, list(self.cat_idxs)], self.batches[x][:, list(self.cont_idxs)]]
        row[0] = tensor(row[0].astype(np.int64)).to(self.device)
        row[1] = tensor(row[1].astype(np.float32)).to(self.device)
        return row
        
    def make_batches(self):
        "Splits data into equal sized batches, excluding the final partial"
        n_splits = len(self.arr)//self.bs
        last = len(self.arr) - (len(self.arr) - (n_splits * self.bs))
        if len(self.arr) > self.bs:
            arrs = np.split(self.arr[:last], n_splits)
            arrs.append(self.arr[last:])
        else:
            arrs = [self.arr]
        self.batches = arrs
        
    def __len__(self): return len(self.arr)//self.bs + (0 if len(self.arr)%self.bs==0 else 1)
show_doc(TabularDataset)
show_doc(TabularDataset.__init__)
show_doc(TabularDataset.make_batches)
#slow
df = pd.read_csv('/home/ml1/.fastai/data/adult_sample/adult.csv')
df = df.head().to_numpy()
df = apply_procs(df, procs)
dset = TabularDataset(df, procs)
#slow
dset[0]
#export
class tabular_learner():
    "A `Learner`-like wrapper for tabular data"
    def __init__(self, data_fn, model_fn):
        "Accepts a `data_fn` and a `model_fn` corresponding to the named picle exports"
        map_location = 'cpu' if not torch.cuda.is_available() else 'cuda'
        self.model = torch.load(model_fn, map_location=map_location)
        self.model.eval()
        with open(data_fn, 'rb') as handle:
            self.procs = pickle.load(handle)
            for proc in self.procs['Categorize']:
                self.procs['Categorize'][proc][np.nan] = 0 # we can't pickle np.nan
            
    def test_dl(self, test_items, bs=64):
        "Applies `procs` to `test_items`"
        dl = apply_procs(test_items, self.procs)
        return TabularDataset(dl, self.procs, bs=bs)
    
    def predict(self, inps):
        "Predict a single tensor"
        with torch.no_grad():
            outs = self.model(*inps)
        outs = np.argmax(outs.cpu().numpy(), axis=1)
        outs = [learn.procs['Outputs'][i] for i in outs]
        return outs
    
    def get_preds(self, dl=None):
        "Predict on multiple batches of data in `dl`"
        outs = []
        for i, batch in enumerate(dl):
            outs += self.predict(batch)
        return outs
show_doc(tabular_learner)
show_doc(tabular_learner.__init__)
#slow
learn = tabular_learner('procs.pkl', 'model.pkl')
show_doc(tabular_learner.test_dl)
#slow
df = pd.read_csv('/home/ml1/.fastai/data/adult_sample/adult.csv')
dl = learn.test_dl(df.iloc[:5].to_numpy())
#slow
dl[0]
show_doc(tabular_learner.predict)
#slow
learn.predict(dl[0])
show_doc(tabular_learner.get_preds)
#slow
learn.get_preds(dl=dl)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13946132464295985, 'fraction_numerical': 0.0060385315824786736, 'mean_word_length': 3.4723531933133303, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '237991', 'license': 'Apache-2.0', 'n_tokens_mistral': 3616, 'n_tokens_neox': 3570, 'n_words': 878, 'path': '01c_transforms.tabular.ipynb', 'repo_name': 'muellerzr/fastinference-pytorch', 'size': '10433'}