How to set a system prompt?
How to set a system prompt? When I do msgs = [{'role': 'system', 'content': "You are an AI"}, {'role': 'user', 'content': "who are you?"}]
I get this error File "/root/.cache/huggingface/modules/transformers_modules/openbmb/MiniCPM-Llama3-V-2_5/26b4c4e55f38ab474b5b8d794bfdd1d5fa5317ce/modeling_minicpmv.py", line 361, in chat
assert role in ["user", "assistant"]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
Code:
class Model:
def __init__(self, **kwargs):
self.model = None
self.tokenizer = None
def load(self):
self.model = AutoModel.from_pretrained(
'openbmb/MiniCPM-Llama3-V-2_5', device_map='auto', trust_remote_code=True, torch_dtype=torch.float16).to("cuda")
self.tokenizer = AutoTokenizer.from_pretrained(
'openbmb/MiniCPM-Llama3-V-2_5', trust_remote_code=True)
def predict(self, model_input):
query = model_input["query"]
image = model_input["image"]
system = model_input["system"]
if image[:5] == "https":
image = Image.open(requests.get(
image, stream=True).raw).convert("RGB")
else:
image = b64_to_pil(image)
with torch.no_grad():
msgs = [{'role': 'system', 'content': "You are an AI"},
{'role': 'user', 'content': "who are you?"}]
res = self.model.chat(
image=image,
msgs=msgs,
tokenizer=self.tokenizer,
sampling=True,
temperature=0.7
)
return {"result": res}
Thank you for your attention! For this version of MiniCPM-Llama3-V-2.5, we don't use a role(system) to set the system prompt, you can just set the system prompt you want at the beginning of the user content~
@Cuiunbo Can you please give me example to do that? How the msgs array should look like?
{'role': 'user', 'content': "my systemprompt" "who are you?"}] Like this? @Cuiunbo
content = []
# message = [
# {'type': 'text', 'value': 'sys prompt'},
# {'type': 'image', 'value': '/path/to/image1.jpg'},
# {'type': 'text', 'value': 'Here is an image:'},
# ]
for x in message:
if x['type'] == 'text':
content.append(x['value'])
elif x['type'] == 'image':
image = Image.open(x['value']).convert('RGB')
content.append(image)
msgs = [{'role': 'user', 'content': content}]
res = self.model.chat(
msgs=msgs,
context=None,
tokenizer=self.tokenizer,
**default_kwargs
)
like thiss
@Cuiunbo It's generating some nonsense response:
def predict(self, model_input):
query = model_input["query"]
image = model_input["image"]
system = model_input["system"]
with torch.no_grad():
response = requests.get(image)
# image = Image.open(BytesIO(response.content)).convert('RGB')
content = []
message = [
{'type': 'text', 'value': system},
{'type': 'image', 'value': '/path/to/image1.jpg'},
{'type': 'text', 'value': query}
]
for x in message:
if x['type'] == 'text':
content.append(x['value'])
elif x['type'] == 'image':
image = Image.open(BytesIO(response.content)).convert('RGB')
content.append(image)
msgs = [{'role': 'user', 'content': content}]
res = self.model.chat(
image=image,
msgs=msgs,
tokenizer=self.tokenizer,
sampling=True,
temperature=0.7
)
return {"result": res}
@Cuiunbo Can you help me fix this system prompt problem?
def predict(self, model_input):
query = model_input["query"]
image = model_input["image"]
system = model_input["system"]
with torch.no_grad():
response = requests.get(image)
# image = Image.open(BytesIO(response.content)).convert('RGB')
content = []
message = [
{'type': 'text', 'value': system},
{'type': 'image', 'value': '/path/to/image1.jpg'},
{'type': 'text', 'value': query}
]
for x in message:
if x['type'] == 'text':
content.append(x['value'])
elif x['type'] == 'image':
image = Image.open(BytesIO(response.content)).convert('RGB')
content.append(image)
############################################
wrong tightening : please edit
msgs = [{'role': 'user', 'content': content}]
res = self.model.chat(
image=image,
msgs=msgs,
tokenizer=self.tokenizer,
sampling=True,
temperature=0.7
)
############################################
return {"result": res}
@Cuiunbo It's hallucinating a lot saying things that are not present in the image. is this model better then COGVLM2?
This model achieves a lower hallucination rate than the GPT4V 20231106 version, but I haven't tried to review CogVLM2's hallucination score, so you can try it out.
With a 19B parameter, it should normally be better in some areas and worse in others than this 8B model.
CogVLM2 is also a good choice if you have more GPU resources.
It's good to compare the two, and I look forward to your comparison!