RuntimeError: Expected query, key, and value to have the same dtype, but got query.dtype: float key.dtype: float and value.dtype: c10::Half instead.
#5
by
Talha
- opened
I am trying to use falcon with langhcain
this is what I am doing
model
model_id='falcon-7b'
tokenizer = AutoTokenizer.from_pretrained(model_id,trust_remote_code=True )
model = AutoModelForCausalLM.from_pretrained(model_id,trust_remote_code=True,
quantization_config = BitsAndBytesConfig(load_in_4bit=True,
bnb_4bit_compute_dtype=torch.bfloat16),
device_map='auto')
pipe = pipeline(
"text-generation",
model=model,
tokenizer=tokenizer,
max_new_tokens=256
)
local_llm = HuggingFacePipeline(pipeline=pipe)
embeddings
hfemb = HuggingFaceEmbeddings()
retriever = vector_db.as_retriever()
memory
memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True,output_key='answer')
chain = ConversationalRetrievalChain.from_llm(local_llm,
retriever=retriever, memory=memory,
chain_type="map_reduce", #
return_source_documents=True)
The code worked with flan-t5 but give error with falcon,
Here is full error
│ [Errno 2] No such file or directory: '/tmp/ipykernel_67349/1702757523.py' │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/base.py:140 in __call__ │
│ │
│ 137 │ │ │ ) │
│ 138 │ │ except (KeyboardInterrupt, Exception) as e: │
│ 139 │ │ │ run_manager.on_chain_error(e) │
│ ❱ 140 │ │ │ raise e │
│ 141 │ │ run_manager.on_chain_end(outputs) │
│ 142 │ │ return self.prep_outputs(inputs, outputs, return_only_outputs) │
│ 143 │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/base.py:134 in __call__ │
│ │
│ 131 │ │ ) │
│ 132 │ │ try: │
│ 133 │ │ │ outputs = ( │
│ ❱ 134 │ │ │ │ self._call(inputs, run_manager=run_manager) │
│ 135 │ │ │ │ if new_arg_supported │
│ 136 │ │ │ │ else self._call(inputs) │
│ 137 │ │ │ ) │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/conversational_retrieval/base.py: │
│ 110 in _call │
│ │
│ 107 │ │ new_inputs = inputs.copy() │
│ 108 │ │ new_inputs["question"] = new_question │
│ 109 │ │ new_inputs["chat_history"] = chat_history_str │
│ ❱ 110 │ │ answer = self.combine_docs_chain.run( │
│ 111 │ │ │ input_documents=docs, callbacks=_run_manager.get_child(), **new_inputs │
│ 112 │ │ ) │
│ 113 │ │ if self.return_source_documents: │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/base.py:239 in run │
│ │
│ 236 │ │ │ return self(args[0], callbacks=callbacks)[self.output_keys[0]] │
│ 237 │ │ │
│ 238 │ │ if kwargs and not args: │
│ ❱ 239 │ │ │ return self(kwargs, callbacks=callbacks)[self.output_keys[0]] │
│ 240 │ │ │
│ 241 │ │ if not kwargs and not args: │
│ 242 │ │ │ raise ValueError( │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/base.py:140 in __call__ │
│ │
│ 137 │ │ │ ) │
│ 138 │ │ except (KeyboardInterrupt, Exception) as e: │
│ 139 │ │ │ run_manager.on_chain_error(e) │
│ ❱ 140 │ │ │ raise e │
│ 141 │ │ run_manager.on_chain_end(outputs) │
│ 142 │ │ return self.prep_outputs(inputs, outputs, return_only_outputs) │
│ 143 │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/base.py:134 in __call__ │
│ │
│ 131 │ │ ) │
│ 132 │ │ try: │
│ 133 │ │ │ outputs = ( │
│ ❱ 134 │ │ │ │ self._call(inputs, run_manager=run_manager) │
│ 135 │ │ │ │ if new_arg_supported │
│ 136 │ │ │ │ else self._call(inputs) │
│ 137 │ │ │ ) │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/combine_documents/base.py:84 in │
│ _call │
│ │
│ 81 │ │ docs = inputs[self.input_key] │
│ 82 │ │ # Other keys are assumed to be needed for LLM prediction │
│ 83 │ │ other_keys = {k: v for k, v in inputs.items() if k != self.input_key} │
│ ❱ 84 │ │ output, extra_return_dict = self.combine_docs( │
│ 85 │ │ │ docs, callbacks=_run_manager.get_child(), **other_keys │
│ 86 │ │ ) │
│ 87 │ │ extra_return_dict[self.output_key] = output │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/combine_documents/map_reduce.py:1 │
│ 44 in combine_docs │
│ │
│ 141 │ │ Combine by mapping first chain over all documents, then reducing the results. │
│ 142 │ │ This reducing can be done recursively if needed (if there are many documents). │
│ 143 │ │ """ │
│ ❱ 144 │ │ results = self.llm_chain.apply( │
│ 145 │ │ │ # FYI - this is parallelized and so it is fast. │
│ 146 │ │ │ [{self.document_variable_name: d.page_content, **kwargs} for d in docs], │
│ 147 │ │ │ callbacks=callbacks, │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/llm.py:157 in apply │
│ │
│ 154 │ │ │ response = self.generate(input_list, run_manager=run_manager) │
│ 155 │ │ except (KeyboardInterrupt, Exception) as e: │
│ 156 │ │ │ run_manager.on_chain_error(e) │
│ ❱ 157 │ │ │ raise e │
│ 158 │ │ outputs = self.create_outputs(response) │
│ 159 │ │ run_manager.on_chain_end({"outputs": outputs}) │
│ 160 │ │ return outputs │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/llm.py:154 in apply │
│ │
│ 151 │ │ │ {"input_list": input_list}, │
│ 152 │ │ ) │
│ 153 │ │ try: │
│ ❱ 154 │ │ │ response = self.generate(input_list, run_manager=run_manager) │
│ 155 │ │ except (KeyboardInterrupt, Exception) as e: │
│ 156 │ │ │ run_manager.on_chain_error(e) │
│ 157 │ │ │ raise e │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/chains/llm.py:79 in generate │
│ │
│ 76 │ ) -> LLMResult: │
│ 77 │ │ """Generate LLM result from inputs.""" │
│ 78 │ │ prompts, stop = self.prep_prompts(input_list, run_manager=run_manager) │
│ ❱ 79 │ │ return self.llm.generate_prompt( │
│ 80 │ │ │ prompts, stop, callbacks=run_manager.get_child() if run_manager else None │
│ 81 │ │ ) │
│ 82 │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/llms/base.py:134 in generate_prompt │
│ │
│ 131 │ │ callbacks: Callbacks = None, │
│ 132 │ ) -> LLMResult: │
│ 133 │ │ prompt_strings = [p.to_string() for p in prompts] │
│ ❱ 134 │ │ return self.generate(prompt_strings, stop=stop, callbacks=callbacks) │
│ 135 │ │
│ 136 │ async def agenerate_prompt( │
│ 137 │ │ self, │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/llms/base.py:191 in generate │
│ │
│ 188 │ │ │ │ ) │
│ 189 │ │ │ except (KeyboardInterrupt, Exception) as e: │
│ 190 │ │ │ │ run_manager.on_llm_error(e) │
│ ❱ 191 │ │ │ │ raise e │
│ 192 │ │ │ run_manager.on_llm_end(output) │
│ 193 │ │ │ return output │
│ 194 │ │ if len(missing_prompts) > 0: │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/llms/base.py:185 in generate │
│ │
│ 182 │ │ │ ) │
│ 183 │ │ │ try: │
│ 184 │ │ │ │ output = ( │
│ ❱ 185 │ │ │ │ │ self._generate(prompts, stop=stop, run_manager=run_manager) │
│ 186 │ │ │ │ │ if new_arg_supported │
│ 187 │ │ │ │ │ else self._generate(prompts, stop=stop) │
│ 188 │ │ │ │ ) │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/llms/base.py:436 in _generate │
│ │
│ 433 │ │ new_arg_supported = inspect.signature(self._call).parameters.get("run_manager") │
│ 434 │ │ for prompt in prompts: │
│ 435 │ │ │ text = ( │
│ ❱ 436 │ │ │ │ self._call(prompt, stop=stop, run_manager=run_manager) │
│ 437 │ │ │ │ if new_arg_supported │
│ 438 │ │ │ │ else self._call(prompt, stop=stop) │
│ 439 │ │ │ ) │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/langchain/llms/huggingface_pipeline.py:159 in │
│ _call │
│ │
│ 156 │ │ stop: Optional[List[str]] = None, │
│ 157 │ │ run_manager: Optional[CallbackManagerForLLMRun] = None, │
│ 158 │ ) -> str: │
│ ❱ 159 │ │ response = self.pipeline(prompt) │
│ 160 │ │ if self.pipeline.task == "text-generation": │
│ 161 │ │ │ # Text generation return includes the starter text. │
│ 162 │ │ │ text = response[0]["generated_text"][len(prompt) :] │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/transformers/pipelines/text_generation.py:201 in │
│ __call__ │
│ │
│ 198 │ │ │ - **generated_token_ids** (`torch.Tensor` or `tf.Tensor`, present when `retu │
│ 199 │ │ │ ids of the generated text. │
│ 200 │ │ """ │
│ ❱ 201 │ │ return super().__call__(text_inputs, **kwargs) │
│ 202 │ │
│ 203 │ def preprocess(self, prompt_text, prefix="", handle_long_generation=None, **generate │
│ 204 │ │ inputs = self.tokenizer( │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/transformers/pipelines/base.py:1118 in __call__ │
│ │
│ 1115 │ │ │ │ ) │
│ 1116 │ │ │ ) │
│ 1117 │ │ else: │
│ ❱ 1118 │ │ │ return self.run_single(inputs, preprocess_params, forward_params, postproces │
│ 1119 │ │
│ 1120 │ def run_multi(self, inputs, preprocess_params, forward_params, postprocess_params): │
│ 1121 │ │ return [self.run_single(item, preprocess_params, forward_params, postprocess_par │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/transformers/pipelines/base.py:1125 in run_single │
│ │
│ 1122 │ │
│ 1123 │ def run_single(self, inputs, preprocess_params, forward_params, postprocess_params): │
│ 1124 │ │ model_inputs = self.preprocess(inputs, **preprocess_params) │
│ ❱ 1125 │ │ model_outputs = self.forward(model_inputs, **forward_params) │
│ 1126 │ │ outputs = self.postprocess(model_outputs, **postprocess_params) │
│ 1127 │ │ return outputs │
│ 1128 │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/transformers/pipelines/base.py:1024 in forward │
│ │
│ 1021 │ │ │ │ inference_context = self.get_inference_context() │
│ 1022 │ │ │ │ with inference_context(): │
│ 1023 │ │ │ │ │ model_inputs = self._ensure_tensor_on_device(model_inputs, device=se │
│ ❱ 1024 │ │ │ │ │ model_outputs = self._forward(model_inputs, **forward_params) │
│ 1025 │ │ │ │ │ model_outputs = self._ensure_tensor_on_device(model_outputs, device= │
│ 1026 │ │ │ else: │
│ 1027 │ │ │ │ raise ValueError(f"Framework {self.framework} is not supported") │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/transformers/pipelines/text_generation.py:263 in │
│ _forward │
│ │
│ 260 │ │ │ │ generate_kwargs["min_length"] += prefix_length │
│ 261 │ │ │
│ 262 │ │ # BS x SL │
│ ❱ 263 │ │ generated_sequence = self.model.generate(input_ids=input_ids, attention_mask=att │
│ 264 │ │ out_b = generated_sequence.shape[0] │
│ 265 │ │ if self.framework == "pt": │
│ 266 │ │ │ generated_sequence = generated_sequence.reshape(in_b, out_b // in_b, *genera │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/torch/utils/_contextlib.py:115 in decorate_context │
│ │
│ 112 │ @functools.wraps(func) │
│ 113 │ def decorate_context(*args, **kwargs): │
│ 114 │ │ with ctx_factory(): │
│ ❱ 115 │ │ │ return func(*args, **kwargs) │
│ 116 │ │
│ 117 │ return decorate_context │
│ 118 │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/transformers/generation/utils.py:1518 in generate │
│ │
│ 1515 │ │ │ │ ) │
│ 1516 │ │ │ │
│ 1517 │ │ │ # 11. run greedy search │
│ ❱ 1518 │ │ │ return self.greedy_search( │
│ 1519 │ │ │ │ input_ids, │
│ 1520 │ │ │ │ logits_processor=logits_processor, │
│ 1521 │ │ │ │ stopping_criteria=stopping_criteria, │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/transformers/generation/utils.py:2335 in │
│ greedy_search │
│ │
│ 2332 │ │ │ model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs) │
│ 2333 │ │ │ │
│ 2334 │ │ │ # forward pass to get next token │
│ ❱ 2335 │ │ │ outputs = self( │
│ 2336 │ │ │ │ **model_inputs, │
│ 2337 │ │ │ │ return_dict=True, │
│ 2338 │ │ │ │ output_attentions=output_attentions, │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1501 in _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/accelerate/hooks.py:165 in new_forward │
│ │
│ 162 │ │ │ with torch.no_grad(): │
│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │
│ 164 │ │ else: │
│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │
│ 166 │ │ return module._hf_hook.post_forward(module, output) │
│ 167 │ │
│ 168 │ module.forward = new_forward │
│ │
│ /home/talha/.cache/huggingface/modules/transformers_modules/falcon-7b/modelling_RW.py:753 in │
│ forward │
│ │
│ 750 │ │ │
│ 751 │ │ return_dict = return_dict if return_dict is not None else self.config.use_return │
│ 752 │ │ │
│ ❱ 753 │ │ transformer_outputs = self.transformer( │
│ 754 │ │ │ input_ids, │
│ 755 │ │ │ past_key_values=past_key_values, │
│ 756 │ │ │ attention_mask=attention_mask, │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1501 in _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/accelerate/hooks.py:165 in new_forward │
│ │
│ 162 │ │ │ with torch.no_grad(): │
│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │
│ 164 │ │ else: │
│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │
│ 166 │ │ return module._hf_hook.post_forward(module, output) │
│ 167 │ │
│ 168 │ module.forward = new_forward │
│ │
│ /home/talha/.cache/huggingface/modules/transformers_modules/falcon-7b/modelling_RW.py:648 in │
│ forward │
│ │
│ 645 │ │ │ │ │ head_mask[i], │
│ 646 │ │ │ │ ) │
│ 647 │ │ │ else: │
│ ❱ 648 │ │ │ │ outputs = block( │
│ 649 │ │ │ │ │ hidden_states, │
│ 650 │ │ │ │ │ layer_past=layer_past, │
│ 651 │ │ │ │ │ attention_mask=causal_mask, │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1501 in _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/accelerate/hooks.py:165 in new_forward │
│ │
│ 162 │ │ │ with torch.no_grad(): │
│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │
│ 164 │ │ else: │
│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │
│ 166 │ │ return module._hf_hook.post_forward(module, output) │
│ 167 │ │
│ 168 │ module.forward = new_forward │
│ │
│ /home/talha/.cache/huggingface/modules/transformers_modules/falcon-7b/modelling_RW.py:385 in │
│ forward │
│ │
│ 382 │ │ residual = hidden_states │
│ 383 │ │ │
│ 384 │ │ # Self attention. │
│ ❱ 385 │ │ attn_outputs = self.self_attention( │
│ 386 │ │ │ layernorm_output, │
│ 387 │ │ │ layer_past=layer_past, │
│ 388 │ │ │ attention_mask=attention_mask, │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1501 in _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ /home/talha/venv/lib/python3.10/site-packages/accelerate/hooks.py:165 in new_forward │
│ │
│ 162 │ │ │ with torch.no_grad(): │
│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │
│ 164 │ │ else: │
│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │
│ 166 │ │ return module._hf_hook.post_forward(module, output) │
│ 167 │ │
│ 168 │ module.forward = new_forward │
│ │
│ /home/talha/.cache/huggingface/modules/transformers_modules/falcon-7b/modelling_RW.py:279 in │
│ forward │
│ │
│ 276 │ │ │ key_layer_ = key_layer.reshape(batch_size, self.num_kv, -1, self.head_dim) │
│ 277 │ │ │ value_layer_ = value_layer.reshape(batch_size, self.num_kv, -1, self.head_di │
│ 278 │ │ │ │
│ ❱ 279 │ │ │ attn_output = F.scaled_dot_product_attention( │
│ 280 │ │ │ │ query_layer_, key_layer_, value_layer_, None, 0.0, is_causal=True │
│ 281 │ │ │ ) │
│ 282 │
╰──────────────────────────────────────────────────────────────────────────────────────────────
Thanks for the report, should be resolved with: https://huggingface.co/tiiuae/falcon-7b/commit/1ba2370c784b56f8b31afc66d5234e8fb40a7209.
Though note that inference with datatypes other than bfloat16 has not been fully validated and may incur some model degradation.
Let us know if you have any more issues.
FalconLLM
changed discussion status to
closed