def load_reference_from_stream(f):
    qids_to_relevant_passageids = {}
    for line in f:
        try:
            sample = json.loads(line.strip())
            qid = sample['question_id']
            if (qid in qids_to_relevant_passageids):
                pass
            else:
                qids_to_relevant_passageids[qid] = []
            for answer_paragraph in sample['answer_paragraphs']:
                qids_to_relevant_passageids[qid].append(answer_paragraph['paragraph_id'])
        except:
            raise IOError(('"%s" is not valid format' % line))
    return qids_to_relevant_passageids
def load_reference_from_stream(f):
    qids_to_relevant_passageids = {}
    for line in f:
        try:
            sample = json.loads(line.strip())
            qid = sample['question_id']
            if (qid in qids_to_relevant_passageids):
                pass
            else:
                qids_to_relevant_passageids[qid] = []
            for answer_paragraph in sample['answer_paragraphs']:
                qids_to_relevant_passageids[qid].append(answer_paragraph['paragraph_id'])
        except:
            raise IOError(('"%s" is not valid format' % line))
    return qids_to_relevant_passageids