<jupyter_start><jupyter_text>Build model<jupyter_code>tokenizer = BertTokenizer.from_pretrained("monologg/bert-base-cased-goemotions-original")
model = BertForMultiLabelClassification.from_pretrained("monologg/bert-base-cased-goemotions-original")
model = model.to('cuda:0')

goemotions = MultiLabelPipeline(
    model=model,
    tokenizer=tokenizer,
    threshold=0.3,
    device=0
)<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>from xml_cleaner import get_transcript_fname_by_id, parse_transcript
# Make dataframe from index
ts_path = os.path.join(data_path, 'transcripts') # path folder of where pm-transcripts are stored
index_file_path = os.path.join(data_path, 'index.csv')
index_df = pd.read_csv(index_file_path)

# Make output dir
if not os.path.exists(data_output_path):
    os.makedirs(data_output_path)<jupyter_output><empty_output><jupyter_text>Iterate<jupyter_code>ts['emotions'] = [goemotions(x)[0] for x in ts['sentences']]
# Choose PM
pm_name = '<NAME>'

# Get all IDs
ts_ids = list(index_df[index_df['pm']==pm_name]['id'].astype(int))

# Iterate
for i, ts_id in enumerate(tqdm(ts_ids)):
    f_out = os.path.join(data_output_path, str(ts_id)+'.pkl')
    if not os.path.exists(f_out):
        ts = parse_transcript(get_transcript_fname_by_id(ts_path, ts_id))
        if ts is not None:
            ts['emotions'] = []
            for sentence_ind, sentence in enumerate(ts['sentences']):
                tokens = tokenizer(sentence)
                if len(tokens['input_ids'])>500:
                    print('Sentence %d in document %d is too long'%(ts_id, sentence_ind))
                else:
                    ts['emotions'].append(goemotions(sentence)[0])

            with open(f_out, 'wb') as f:
                pickle.dump(ts, f)<jupyter_output>0%|          | 0/5865 [00:00<?, ?it/s]/home/mahasen/thaum/hackathons/govhack_2020/language-of-leadership/lib/utils/xml_cleaner.py:47: GuessedAtParserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system ("lxml"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.

The code that caused this warning is on line 47 of the file /home/mahasen/thaum/hackathons/govhack_2020/language-of-leadership/lib/utils/xml_cleaner.py. To get rid of this warning, pass the additional argument 'features="lxml"' to the BeautifulSoup constructor.

  soup = bs(xml_file)
Token indices sequence length is longer than the specified maximum sequence length for this model (866 > 512). Running this sequence through the model will result in indexing errors
 24%|██▍       | 1409/5865 [00:11<00:36, 121.44it/s]<jupyter_text>Repeat using Ekman Taxonomy<jupyter_code>from transformers import BertTokenizer
from model import BertForMultiLabelClassification
from multilabel_pipeline import MultiLabelPipeline
from pprint import pprint

tokenizer = BertTokenizer.from_pretrained("monologg/bert-base-cased-goemotions-ekman")
model = BertForMultiLabelClassification.from_pretrained("monologg/bert-base-cased-goemotions-ekman")

goemotions = MultiLabelPipeline(
    model=model,
    tokenizer=tokenizer,
    threshold=0.3
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10611620795107034, 'fraction_numerical': 0.018960244648318043, 'mean_word_length': 4.66897746967071, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '468271', 'license': 'MIT', 'n_tokens_mistral': 1141, 'n_tokens_neox': 1061, 'n_words': 247, 'path': 'nb/GetEmotions.ipynb', 'repo_name': 'thaum-io/language-of-leadership', 'size': '3276'}