<jupyter_start><jupyter_text>Inference> Provides faster inference scripts<jupyter_code>#hide
from nbdev.showdoc import *
#export
from fastinference.soft_dependencies import SoftDependencies
if not SoftDependencies.check()['light']:
    raise ImportError("The fastai module is not installed.")
#export
from fastai.learner import _ConstantFunc
from fastai.vision.all import *
from fastai.tabular.all import *
#export
@typedispatch
def _fully_decode(dl:TfmdDL, inps, outs):
    "Attempt to fully decode the `inp"
    inps = dl.decode(inps)
    dec = []
    for d in inps:
        for e in d:
            e = retain_type(e, typ=type(d))
            dec.append(e)
    outs.insert(len(outs), dec)
    return outs

@typedispatch
def _fully_decode(dl:TabDataLoader, inps, outs):
    "Attempt to fully decode the `inp"
    for i in range(dl.n_inp):
        inps[i] = torch.cat(inps[i], dim=0)
    dec = dl.decode(inps)
    outs.insert(len(outs), dec)
    return outs

def _decode_loss(vocab, dec_out, outs):
    i2c, dec2c = {}, []
    try:
        for i, vocab in enumerate(list(vocab)):
                i2c[i] = vocab
        for i, item in enumerate(dec_out):
            dec2c.append(i2c[int(item)])
        dec_out = dec2c
        outs.insert(0, dec_out)
    except:
        outs.insert(0, dec_out)
    return outs
#export
@delegates(GatherPredsCallback.__init__)
@patch
def get_preds(self:Learner, ds_idx=1, dl=None, with_input=False, with_decoded=False, with_loss=False, raw=False, act=None,
                inner=False, reorder=True, cbs=None, **kwargs):
    if dl is None: dl = self.dls[ds_idx].new(shuffled=False, drop_last=False)
    if reorder and hasattr(dl, 'get_idxs'):
        idxs = dl.get_idxs()
        dl = dl.new(get_idxs = _ConstantFunc(idxs))
    cb = GatherPredsCallback(with_input=with_input, with_loss=with_loss, **kwargs)
    ctx_mgrs = self.validation_context(cbs=L(cbs)+[cb], inner=inner)
    if with_loss: ctx_mgrs.append(self.loss_not_reduced())
    with ContextManagers(ctx_mgrs):
        self._do_epoch_validate(dl=dl)
        if act is None: act = getattr(self.loss_func, 'activation', noop)
        res = cb.all_tensors()
        pred_i = 1 if with_input else 0
        if res[pred_i] is not None:
            res[pred_i] = act(res[pred_i]) if not raw else res[pred_i]
            if with_decoded: 
                res.insert(pred_i+2, getattr(self.loss_func, 'decodes', noop)(res[pred_i]))
        if reorder and hasattr(dl, 'get_idxs'): res = nested_reorder(res, tensor(idxs).argsort())
        if with_decoded:
            if hasattr(self.dls, 'categorize'): res[pred_i+1] = apply(getattr(self.dls, 'categorize', noop).decode, [*res[pred_i+2]])
            if hasattr(self.dls, 'multi_categorize'): res[pred_i+1] = apply(getattr(self.dls, 'multi_categorize', noop).decode, [*res[pred_i+2]])
        return tuple(res)
    self._end_cleanup()
show_doc(Learner.get_preds)<jupyter_output><empty_output><jupyter_text>This function is *almost* the exact same as `fastai`'s. The big difference is we can return our raw outputs or our class names.> Note: to speed up inference, multi-processing *will* slow you down. In production use `num_workers=0` unless you have a large amount of vision data being passed at one time<jupyter_code>#export
@patch
def predict(x:Learner, item, with_input=False, rm_type_tfms=None):
        dl = x.dls.test_dl([item], rm_type_tfms=rm_type_tfms, num_workers=0)
        res = x.get_preds(dl=dl, with_input=with_input, with_decoded=True)
        return res
#export
@patch
def predict(x:TabularLearner, row, with_input=False, rm_type_tfms=None):
        dl = x.dls.test_dl(row.to_frame().T)
        dl.dataset.conts = dl.dataset.conts.astype(np.float32)
        res = x.get_preds(dl=dl, with_decoded=True, with_input=with_input)
        return res<jupyter_output><empty_output><jupyter_text>ExamplesBelow is examples with `Vision`, `Tabular`, and `Text`:  VisionWe'll use the `PETs` dataset:<jupyter_code>path = untar_data(URLs.PETS)
fnames = get_image_files(path/'images')
pat = r'(.+)_\d+.jpg$'
batch_tfms = [*aug_transforms(size=224, max_warp=0), Normalize.from_stats(*imagenet_stats)]
item_tfms = RandomResizedCrop(460, min_scale=0.75, ratio=(1.,1.))
bs=64
dls = ImageDataLoaders.from_name_re(path, fnames, pat, batch_tfms=batch_tfms, 
                                   item_tfms=item_tfms, bs=bs)
learn = cnn_learner(dls, resnet18, metrics=accuracy)
o = learn.predict(fnames[0], with_input=False)
o
imgs, probs, classes, clas_idx = learn.get_preds(dl=learn.dls.test_dl(fnames[:3]), with_input=True, with_decoded=True)<jupyter_output><empty_output><jupyter_text>The first index will contain our **raw** transformed images:<jupyter_code>TensorImage(imgs[0]).show()<jupyter_output>Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).<jupyter_text>Second the probabilities<jupyter_code>probs[0]<jupyter_output><empty_output><jupyter_text>And third our class names:<jupyter_code>classes[0]<jupyter_output><empty_output><jupyter_text>TabularWe'll use the `ADULT_SAMPLE` dataset:<jupyter_code>from fastai.tabular.all import *
path = untar_data(URLs.ADULT_SAMPLE)
df = pd.read_csv(path/'adult.csv')
splits = RandomSplitter()(range_of(df))
cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']
cont_names = ['age', 'fnlwgt', 'education-num']
procs = [Categorify, FillMissing, Normalize]
y_names = 'salary'
to = TabularPandas(df, procs=procs, cat_names=cat_names, cont_names=cont_names,
                   y_names=y_names, splits=splits)
dls = to.dataloaders()
learn = tabular_learner(dls, layers=[200,100])
inp, probs, name, cat_idx = learn.predict(df.iloc[0], with_input=True)
inp<jupyter_output><empty_output><jupyter_text>For tabular, our input is a `TabularPandas` row. We can decode that row utilizing the `DataLoaders` `decode` function to generate it, and show it:<jupyter_code>learn.dls.decode(inp).show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1277341793287694, 'fraction_numerical': 0.009183503088996493, 'mean_word_length': 4.525830258302583, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '51207', 'license': 'Apache-2.0', 'n_tokens_mistral': 2243, 'n_tokens_neox': 2194, 'n_words': 462, 'path': 'nbs/00_inference.ipynb', 'repo_name': 'hududed/fastinference', 'size': '5989'}