update FIM tokens and examples
Browse filesbased on https://huggingface.co/bigcode/tiny_starcoder_py/discussions/2#64734c6e16ef9c21dba65637
README.md
CHANGED
@@ -61,7 +61,7 @@ print(tokenizer.decode(outputs[0]))
|
|
61 |
Fill-in-the-middle uses special tokens to identify the prefix/middle/suffix part of the input and output:
|
62 |
|
63 |
```python
|
64 |
-
input_text = "<
|
65 |
inputs = tokenizer.encode(input_text, return_tensors="pt").to(device)
|
66 |
outputs = model.generate(inputs)
|
67 |
print(tokenizer.decode(outputs[0]))
|
|
|
61 |
Fill-in-the-middle uses special tokens to identify the prefix/middle/suffix part of the input and output:
|
62 |
|
63 |
```python
|
64 |
+
input_text = "<fim_prefix>def print_one_two_three():\n print('one')\n <fim_suffix>\n print('three')<fim_middle>"
|
65 |
inputs = tokenizer.encode(input_text, return_tensors="pt").to(device)
|
66 |
outputs = model.generate(inputs)
|
67 |
print(tokenizer.decode(outputs[0]))
|