Update inference.py

inference.py

@@ -360,7 +360,8 @@ def load_model(ckpt_dir, device):
     # model = fabric.setup(model)
     state_dict = lazy_load(ckpt_dir + "/lit_model.pth")
     model.load_state_dict(state_dict, strict=True)
-    model.to(device).eval()
+    model = model.to(device)
+    model.eval()
 
     return fabric, model, text_tokenizer, snacmodel, whispermodel