Frederic Marvin Abraham commited on
Commit
5d526dc
1 Parent(s): a2a8837

add custom handler

Browse files
Files changed (1) hide show
  1. handler.py +27 -0
handler.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Dict, List, Any
2
+
3
+ import torch
4
+ from transformers import BertModel, BertTokenizerFast
5
+
6
+
7
+ class EndpointHandler():
8
+ def __init__(self, path_to_model: str):
9
+ # Preload all the elements you are going to need at inference.
10
+ # pseudo:
11
+ self.tokenizer = BertTokenizerFast.from_pretrained(path_to_model)
12
+ self.model = BertModel.from_pretrained(path_to_model)
13
+ self.model = self.model.eval()
14
+
15
+ def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
16
+ """
17
+ This method is called whenever a request is made to the endpoint.
18
+ :param data: { inputs [str]: list of strings to be encoded }
19
+ :return: A :obj:`list` | `dict`: will be serialized and returned
20
+ """
21
+
22
+ inputs = self.tokenizer(data['inputs'], return_tensors = "pt", padding = True)
23
+
24
+ with torch.no_grad():
25
+ outputs = self.model(**inputs)
26
+
27
+ return outputs.pooler_output