<jupyter_start><jupyter_text>Question Generation<jupyter_code># collect all paragraphs
path = '../data/cast19/test_paragraphs.tsv'
qp_ids = {}
paras = {}
with open(path, encoding='utf-8') as f:
    for i, line in enumerate(f):
        q_id, para_id, para_text = line.rstrip().split('\t')
        if q_id not in qp_ids:
            qp_ids[q_id] = []
        qp_ids[q_id].append(para_id)
        if para_id not in paras:
            paras[para_id] = para_text
print(len(paras))
print(qp_ids)
# collect all GS questions: independent
path_qs = '../data/cast19/human_questions.txt'

gs_qs = {}
with open(path_qs, 'r') as f:
    next(f)
    for line in f:
        q_id, q = line.split('\t')
#         print(q_id)
        gs_qs[q_id] = (q.strip('\n'))
print(len(gs_qs))
print(gs_qs['1_11'])
# test rouge scorer
from rouge_score import rouge_scorer

scorer = rouge_scorer.RougeScorer(['rouge1', 'rougeL'], use_stemmer=True)

scores = scorer.score("what is fsa health plan .", "what is health care fsa plan .")  # reference predicted
print(scores)<jupyter_output>{'rouge1': Score(precision=0.8333333333333334, recall=1.0, fmeasure=0.9090909090909091), 'rougeL': Score(precision=0.6666666666666666, recall=0.8, fmeasure=0.7272727272727272)}<jupyter_text>Global ContextGlobal context means sequences of questions from the same conversation. Incorporate global awareness when selecting next questions with respect to the previously selected questions.<jupyter_code># group question ids into sequences within the same conversation
from collections import defaultdict

question_sequences = defaultdict(list)
for q_id in qp_ids:
    conversation_id = q_id.split('_')[0]
    question_sequences[conversation_id].append(q_id)
#     print(gs_qs[q_id])
print(question_sequences)
# load generated questions into a dictionary
import os
import json

generated_questions = defaultdict(list)
questions_specificity = defaultdict(list)

squash_path = '../data/cast19/squash'

sim_threshold = 0.8  # rouge similarity threshold for duplicate questions

for question_id in os.listdir(squash_path):
    print(question_id)
    try:
        with open(squash_path+'/%s/generated_questions.json' % question_id) as f:
            paragraphs = json.load(f)['data'][0]['paragraphs']
            # group all generated questions to reference questions
            for i, p in enumerate(paragraphs):
                qs, ss = [], []
                for qa in p['qas']:
#                     print(qa['class'])
                    # do not keep duplicate questions for the same paragraph
                    q = qa['question']#.lower()
                    if q not in qs:
                        # check similarity to existing questions
                        duplicate = False
                        for q_ref in qs:
        #                     print(q_ref)
                            scores = scorer.score(q_ref, q)
                            rouge1f = scores['rouge1'].fmeasure
                            if rouge1f > sim_threshold:
                                duplicate = True
                                break
        #                     print(rouge1f)
                        if not duplicate:
                            qs.append(q)
                            ss.append(qa['class'])
                generated_questions[question_id].extend(qs)
                questions_specificity[question_id].extend(ss)
#                 print(len(qs))
#         print(len(generated_questions[question_id]))
#         break
    except:
        pass

print(len(generated_questions))       
# question selection with embedding based similarity
from sklearn.metrics.pairwise import cosine_similarity

# load best model evaluated on STSbenchmark fine-tuned on semantic textual similarity task using AllNLI and STSbenchmark
from sentence_transformers import SentenceTransformer
model = SentenceTransformer('roberta-large-nli-stsb-mean-tokens')

# evaluate squash baseline wrt question sequences
import numpy as np

mean_scores = []

for sequence_id, q_ids in question_sequences.items():
#     if sequence_id != '40':
#         continue
    print(sequence_id)
    
    # rouge scores per conversation
    scores = []
    # interate over questions in a sequence
    generated_sequence = []
    for q_id in q_ids:
        
        # get reference question per id
        q_ref = gs_qs[q_id]
#         print("Reference:", q_ref)
        
        # get generated questions per id
        q_gens = generated_questions[q_id]
        q_specs = questions_specificity[q_id]
        if q_gens:
            # encode all questions as vectors (embed)
            q_vectors = model.encode(q_gens)
    #         print(len(q_vectors))

            # compare all questions to each other to select a centroid (most similar question)
            similarities_matrix = cosine_similarity(q_vectors, q_vectors)
            sum_similarities = sum(similarities_matrix).tolist()  # aggregate scores across all questions

            while True:
                # calculate the most similar (representative) question (centroid)
                max_score = max(sum_similarities)
                max_index = np.argmax(sum_similarities)
                q = q_gens[max_index]
                # skip duplicate questions
                if q in generated_sequence:
                    del sum_similarities[max_index]
                    del q_gens[max_index]
                    continue
                break
#                 print("Centroid:", q)
            # report on question specificity
            s = q_specs[max_index]
            # calculate score of the selected question to the reference question
            score = scorer.score(q_ref, q)  # reference predicted
    #         print(score)
            # report sample
            print("%s -> %s %.2f" % (q_ref, q, score['rouge1'].fmeasure), s)

            # accumulate generated questions and scores
            generated_sequence.append(q)
            scores.append(score['rouge1'].fmeasure)
    
    if scores:
        mean_scores.append(np.mean(scores))
#     break
print(len(mean_scores))
print(np.mean(mean_scores))
# question selection oracle
from sklearn.metrics.pairwise import cosine_similarity

# load best model evaluated on STSbenchmark fine-tuned on semantic textual similarity task using AllNLI and STSbenchmark
from sentence_transformers import SentenceTransformer
model = SentenceTransformer('roberta-large-nli-stsb-mean-tokens')

# evaluate squash baseline wrt question sequences
import numpy as np

mean_scores = []
sequences = []
for sequence_id, q_ids in question_sequences.items():
#     if sequence_id != '40':
#         continue
    print(sequence_id)
    
    # rouge scores per conversation
    scores = []
    # interate over questions in a sequence
    generated_sequence = []
    sequence = []
    for q_id in q_ids:
        
        # get reference question per id
        q_ref = gs_qs[q_id]
#         print("Reference:", q_ref)
        
        # get generated questions per id
        q_gens = generated_questions[q_id]
        q_specs = questions_specificity[q_id]
        if q_gens:
            all_q_gens_scores = []
            for q in q_gens:
                _scores = scorer.score(q_ref, q)  # reference predicted
                rouge1f = _scores['rouge1'].fmeasure
        #         print(scores['rouge1'].fmeasure)
                all_q_gens_scores.append(_scores['rouge1'].fmeasure)
            
            while True:
                # calculate the most similar (representative) question (centroid)
                max_score = max(all_q_gens_scores)
                max_index = np.argmax(all_q_gens_scores)
                q = q_gens[max_index]
                
                # skip duplicate questions
                if q.lower() in generated_sequence:
                    del all_q_gens_scores[max_index]
                    del q_gens[max_index]
                    continue
                break
#                 print("Centroid:", q)
    
            # report on question specificity
            s = q_specs[max_index]
            # calculate score of the selected question to the reference question
            score = scorer.score(q_ref, q)  # reference predicted
    #         print(score)
            # report sample
            print("%s -> %s %.2f" % (q_ref, q, score['rouge1'].fmeasure), s)
            
            # accumulate generated questions and scores
            generated_sequence.append(q.lower())
            scores.append(score['rouge1'].fmeasure)
            sequence.append(s)
    
    
    if scores:
        mean_scores.append(np.mean(scores))
        sequences.apppend(sequence)
#     break
print(len(mean_scores))
print(np.mean(mean_scores))
# compare question specificity sequence patterns for human vs generated questions
# question sequences -> keywords / specificity
for s in sequences:
    print(s)
# use keyword-based or specificity-based templates to select questions<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0846230931967487, 'fraction_numerical': 0.011802694577441266, 'mean_word_length': 2.211297819091884, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '76122', 'license': 'MIT', 'n_tokens_mistral': 2651, 'n_tokens_neox': 2473, 'n_words': 655, 'path': 'src/evaluate_squash_baseline-conversational.ipynb', 'repo_name': 'svakulenk0/conversation_generator', 'size': '8981'}