File size: 2,560 Bytes
8795149 ab4986a 9a08573 8795149 2ca3aaa deaf77e 2ca3aaa deaf77e 2ca3aaa deaf77e 2ca3aaa deaf77e 2ca3aaa deaf77e afe4793 2ca3aaa e082688 2ca3aaa deaf77e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 |
---
license: apache-2.0
language:
- he
library_name: transformers
pipeline_tag: token-classification
datasets:
- HeTree/MevakerConcSen
---
## Hebrew Conclusion Extraction Model (based on token classification)
#### How to use
```python
from transformers import RobertaTokenizerFast, AutoModelForTokenClassification
from datasets import load_dataset
def split_into_windows(examples):
return {'sentences': [examples['sentence']], 'labels': [examples["label"]]}
def concatenate_dict_value(dict_obj):
concatenated_dict = {}
for key, value in dict_obj.items():
flattened_list = []
for sublist in value:
if len(flattened_list) + len(sublist) <= 512:
for item in sublist:
flattened_list.append(item)
else:
print("Not all sentences were processed due to length")
break
concatenated_dict[key] = flattened_list
return concatenated_dict
def tokenize_and_align_labels(examples):
tokenized_inputs = tokenizer(examples["sentences"], truncation=True, max_length=512)
tokeized_inp_concat = concatenate_dict_value(tokenized_inputs)
tokenized_inputs["input_ids"] = tokeized_inp_concat['input_ids']
tokenized_inputs["attention_mask"] = tokeized_inp_concat['attention_mask']
word_ids = tokenized_inputs["input_ids"]
labels = []
count = 0
for word_idx in word_ids:
if word_idx == 2:
labels.append(examples[f"labels"][count])
count = count + 1
else:
labels.append(-100)
tokenized_inputs["labels"] = labels
return tokenized_inputs
model = AutoModelForTokenClassification.from_pretrained('HeTree/HeConE')
tokenizer = RobertaTokenizerFast.from_pretrained('HeTree/HeConE')
raw_dataset = load_dataset('HeTree/MevakerConcSen')
window_size = 5
raw_dataset_window = raw_dataset.map(split_into_windows, batched=True, batch_size=window_size, remove_columns=raw_dataset['train'].column_names)
tokenized_dataset = raw_dataset_window.map(tokenize_and_align_labels, batched=False)
```
### Citing
If you use HeConE in your research, please cite [Mevaker: Conclusion Extraction and Allocation Resources for the Hebrew Language](https://arxiv.org/abs/2403.09719).
```
@article{shalumov2024mevaker,
title={Mevaker: Conclusion Extraction and Allocation Resources for the Hebrew Language},
author={Vitaly Shalumov and Harel Haskey and Yuval Solaz},
year={2024},
eprint={2403.09719},
archivePrefix={arXiv},
primaryClass={cs.CL}
}
``` |