<commit_before>#!/usr/bin/env python2.7

from bs4 import BeautifulSoup
import sys

with open(sys.argv[1]) as xmlfile:
    dd = BeautifulSoup(xmlfile, 'xml')
    for psg in dd.find_all('passage'):
        subtopic = psg.parent
        topic = subtopic.parent
        domain = topic.parent

        topic_id = topic['id']
        subtopic_id = subtopic['id']
        doc_id = psg.docno.text
        rating = psg.rating.text

        print "\t".join((topic_id, subtopic_id, doc_id, rating))
        <commit_msg>Dont output multiple docids for multiple passages in XML.<commit_after>#!/usr/bin/env python2.7

from bs4 import BeautifulSoup
import sys
import collections

qrels = collections.OrderedDict()

with open(sys.argv[1]) as xmlfile:
    dd = BeautifulSoup(xmlfile, 'xml')
    for psg in dd.find_all('passage'):
        subtopic = psg.parent
        topic = subtopic.parent
        domain = topic.parent

        topic_id = topic['id']
        subtopic_id = subtopic['id']
        doc_id = psg.docno.text
        rating = psg.rating.text

        qrels[topic_id + '\t' + subtopic_id + '\t' + doc_id] = rating

for key in qrels:
    print key, '\t', qrels[key]
{'fraction_non_alphanumeric': 0.11627906976744186, 'fraction_numerical': 0.0068906115417743325, 'mean_word_length': 2.6540880503144653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1957216', 'n_tokens_mistral': 404, 'n_tokens_neox': 386, 'n_words': 97}