<jupyter_start><jupyter_text>Overview This is the code I was originally using to identify sentences in the text associated with questions in the SQuAD. The goal was to train an algorithm to ID key sentences containing the key ideas, since these would be the ones mostly associated with questions.Unfortunately, the majority of the sentences in the document corpus contained questions associated with them, rendering this as containing little information value. Import stuff and load the data===<jupyter_code># Set up and load data
# Includes
import sys
import os
import numpy as np
import json
import os


# Setup paths containing utility
curr_folder = os.getcwd()
sys.path.insert(0, os.path.join(curr_folder,'../app'))

# Load the data
from utils import load_SQuAD_train
arts = load_SQuAD_train()






# All articles
Narticles = len(arts)
print ("Narticles = " +  str(Narticles))<jupyter_output>Narticles = 442<jupyter_text>List titles of all articles===<jupyter_code># List all article titles
print ("Narticles = " +  str(Narticles)) 
for ind in range(Narticles):
    art = arts[ind]
    print(art['title'])<jupyter_output>Narticles = 442
Beyoncé
Frédéric_Chopin
Sino-Tibetan_relations_during_the_Ming_dynasty
IPod
The_Legend_of_Zelda:_Twilight_Princess
Spectre_(2015_film)
2008_Sichuan_earthquake
New_York_City
To_Kill_a_Mockingbird
Solar_energy
Kanye_West
Buddhism
American_Idol
Dog
2008_Summer_Olympics_torch_relay
Genome
Comprehensive_school
Republic_of_the_Congo
Prime_minister
Institute_of_technology
Wayback_Machine
Dutch_Republic
Symbiosis
Canadian_Armed_Forces
Cardinal_(Catholicism)
Iranian_languages
Lighting
Separation_of_powers_under_the_United_States_Constitution
Architecture
Human_Development_Index
Southern_Europe
BBC_Television
Arnold_Schwarzenegger
Plymouth
Heresy
Warsaw_Pact
Materialism
Christian
Sony_Music_Entertainment
Oklahoma_City
Hunter-gatherer
United_Nations_Population_Fund
Russian_Soviet_Federative_Socialist_Republic
Alexander_Graham_Bell
Pub
Internet_service_provider
Comics
Saint_Helena
Aspirated_consonant
Hydrogen
Space_Race
Web_browser
BeiDou_Navigation_Satellite_System
Canon_law
Commu[...]<jupyter_text>Now explore an individual article===<jupyter_code>art = arts[15]
print(art['title'])

i_para = 0
paragraphs = art['paragraphs']
Npara = len(paragraphs)
paragraph = paragraphs[0]
#print(paragraph.keys())
print("Nparagraphs: " + str(Npara))
context = paragraph['context']
print("PARAGRAPH " + str(i_para) + ": " + context)
qs = paragraph['qas']
Nquestions = len(qs)
print("Number of questions: " + str(Nquestions))

i_question = 0
q = qs[i_question]
print("QUESTION: " + str(i_question))
print(q)

qas = q['answers']
Nanswers = len(qas)
print("Number of answers " + str(Nanswers))
i_answer = 0
qa = qas[i_answer]
print(qa)
print('CONTEXT')
print(context)
print('\nQUESTION')
print(q)
print('\nANSWER')
answer_start = qa['answer_start']
print("Starting position " + str(answer_start))
#print()
print(context[answer_start:])<jupyter_output>CONTEXT
In modern molecular biology and genetics, the genome is the genetic material of an organism. It consists of DNA (or RNA in RNA viruses). The genome includes both the genes and the non-coding sequences of the DNA/RNA.

QUESTION
{'question': 'In what fields of science is the genome studied?', 'id': '56dc544814d3a41400c267bf', 'answers': [{'text': 'molecular biology and genetics', 'answer_start': 10}], 'is_impossible': False}

ANSWER
Starting position 10
molecular biology and genetics, the genome is the genetic material of an organism. It consists of DNA (or RNA in RNA viruses). The genome includes both the genes and the non-coding sequences of the DNA/RNA.<jupyter_text>Now want to ID sentence numbers associated with each question--<jupyter_code>import nltk
from nltk.tokenize import sent_tokenize
sentences = sent_tokenize(context)
Nsent = len(sentences)
print(context + "\n")
print(sentences[0])
context[context.find(sentences[2]):]
# Build list of sentence indices
curr = 0
sstart = list()
sstop = list()
for i in range(Nsent):
    sstart.append(curr)
    sstop.append(curr + len(sentences[i]))
    curr = sstop[i] + 1

# Test that it worked
i=0; print(context[sstart[i]:sstop[i]])
i=1; print(context[sstart[i]:sstop[i]])
i=2; print(context[sstart[i]:sstop[i]])
# Add entry to answer describing which question in context it's associated with
Nquestions = len(qs)
for i in range(Nquestions):
    Nanswers = len(qs[i]['answers'])
    for j in range(Nanswers):
        for k in range(len(sstart)):
            if sstart[k] <= qs[i]['answers'][j]['answer_start'] <= sstop[k]:
                qs[i]['answers'][j]['sentence_num'] = k
                break
    
        

# Test that it works
print(context); print('\n')
qnum = 2
print(qs[qnum]); print('\n')

def print_question_context(q):
    if not q['is_impossible']:
        sentence_num = q['answers'][0]['sentence_num']
        char_start = q['answers'][0]['answer_start']
        print("Predicted sentence: " + sentences[sentence_num])
        print("Actual sentence tail: " + sent_tokenize(context[char_start:])[0])

print_question_context(qs[qnum])


# Now start building a list of questions for each sentence

# For creating a list of lists (https://stackoverflow.com/questions/5518435/python-fastest-way-to-create-a-list-of-n-lists)
from itertools import repeat

# Create empty list of lenght Nsentences
Nsentences = len(sentences)
sentences_questions = [[] for i in repeat(None, Nsentences)]


# Pipe the questions appropriately
for q in qs:
    for a in q['answers']:
        sentences_questions[a['sentence_num']].append(q)

sentences_questions
# Build a list for each sentence containing ONLY the number of answers

# Create empty list of lenght Nsentences
Nsentences = len(sentences)
sentences_Nquestions = [0] * Nsentences


# Pipe the questions appropriately
for q in qs:
    for a in q['answers']:
        sentences_Nquestions[a['sentence_num']] = sentences_Nquestions[a['sentence_num']] + 1

print(sentences_Nquestions)<jupyter_output>[1, 2, 0]<jupyter_text>Put it all together - combine into single code block==<jupyter_code># All articles
arts = data['data']
Narticles = len(arts)
print ("Narticles = " +  str(Narticles)) 


art = arts[15]
print(art['title'])

i_para = 9
paragraphs = art['paragraphs']
Npara = len(paragraphs)
paragraph = paragraphs[i_para]
#print(paragraph.keys())
print("Nparagraphs: " + str(Npara))

context = paragraph['context']
print("PARAGRAPH " + str(i_para) + ": " + context)

qs = paragraph['qas']
Nquestions = len(qs)
print("Number of questions: " + str(Nquestions))

import nltk
from nltk.tokenize import sent_tokenize
sentences = sent_tokenize(context)
Nsent = len(sentences)

# Build list of sentence indices
curr = 0
sstart = list()
sstop = list()
for i in range(Nsent):
    sstart.append(curr)
    sstop.append(curr + len(sentences[i]))
    curr = sstop[i] + 1


# Add entry to answer describing which question in context its associated with
Nquestions = len(qs)
for i in range(Nquestions):
    Nanswers = len(qs[i]['answers'])
    for j in range(Nanswers):
        for k in range(len(sstart)):
            if sstart[k] <= qs[i]['answers'][j]['answer_start'] <= sstop[k]:
                qs[i]['answers'][j]['sentence_num'] = k
                break

# Now start building a list of questions for each sentence

# For creating a list of lists (https://stackoverflow.com/questions/5518435/python-fastest-way-to-create-a-list-of-n-lists)
from itertools import repeat

# Create empty list of lenght Nsentences
Nsentences = len(sentences)
sentences_questions = [[] for i in repeat(None, Nsentences)]


# Pipe the questions appropriately
for q in qs:
    for a in q['answers']:
        sentences_questions[a['sentence_num']].append(q)

    
# Build a list for each sentence containing ONLY the number of answers
# Create empty list of lenght Nsentences
Nsentences = len(sentences)
sentences_Nquestions = [0] * Nsentences


# Pipe the questions appropriately
for q in qs:
    for a in q['answers']:
        sentences_Nquestions[a['sentence_num']] = sentences_Nquestions[a['sentence_num']] + 1

#
print(sentences_Nquestions)


print(sentences_questions[3])<jupyter_output>[{'question': 'What is the science that deals with the common genetic characteristics of related organisms called?', 'id': '56dc561414d3a41400c267d3', 'answers': [{'text': 'genomics', 'answer_start': 656, 'sentence_num': 3}], 'is_impossible': False}, {'question': 'In contrast to genomics, genetics usually studies what?', 'id': '56dc561414d3a41400c267d4', 'answers': [{'text': 'single genes or groups of genes', 'answer_start': 745, 'sentence_num': 3}], 'is_impossible': False}]
{'fraction_non_alphanumeric': 0.11144543455618562, 'fraction_numerical': 0.014813100335609305, 'mean_word_length': 5.492862509391435, 'pattern_counts': {'":': 2, '<': 20, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '385444', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3061, 'n_tokens_neox': 2812, 'n_words': 805, 'path': 'play/old/test_important_sentence_ID.ipynb', 'repo_name': 'davestanley/animated-succotash', 'size': '8641'}