<commit_before>from sys import argv

dic = {}

def getEntries(word):
    try:
        entries = {}
        for entry in dic[word]:
            info = entry.split(',')
            locs = {}
            for loc in info[1:]:
                line,word = loc.split(':')
                locs[line] = word
            entries[info[0]] = locs
        return entries        
    except: 
        return None 
    
def printEntries(entries):
    for entry in entries:
        print "  File: {}".format(entry)
        for loc in entries[entry]:
            print "    Line: {}, Word {}".format(loc, entries[entry][loc])
    

if __name__ == '__main__':
    try:
        filename = argv[1]
    except:
        print "use: python query.py indexfile.txt"
        exit()

    with open(filename) as f:
        for line in f:
            entry = line.rstrip().split('\t')
            dic[entry[0]] = entry[1:]

    while True:
        try:
            print "Enter your query:",
            query = raw_input().rstrip().lower()
        except:
            break
        entries = getEntries(query)
        if entries is None:
            print "  {} not found".format(query)
        else:
            printEntries(entries)<commit_msg>query now properly handles the same word appearing multiple times on the same line and can parse entire phrases<commit_after>from sys import argv

class QueryHandler:
    def __init__(self, filename):
        self.dic = {}
        with open(filename) as f:
            for line in f:
                entry = line.rstrip().split('\t')
                self.dic[entry[0]] = entry[1:]

    def getEntries(self,word):
        try:
            entries = {}
            for entry in self.dic[word]:
                info = entry.split(',')
                locs = {}
                for loc in info[1:]:
                    line,word = loc.split(':')
                    if line in locs:
                        locs[line].append(word)
                    else:
                        locs[line] = [word]
                entries[info[0]] = locs
            return entries        
        except: 
            return None 
    
    def printEntries(self,entries):
        for entry in entries:
            print "  File: {}".format(entry)
            for loc in entries[entry]:
                if len(entries[entry][loc]) == 1:
                    print "    Line: {}, Word {}"\
                        .format(loc, entries[entry][loc][0])
                else:
                    print "    Line: {}, Words {}"\
                        .format(loc, ','.join(entries[entry][loc]))
    
    def readQueries(self):
        while True:
            try:
                print "Enter your query:",
                query = raw_input().rstrip().lower()
            except:
                break

            entries = self.getEntries(query)
            if entries is None:
                print "  {} not found".format(query)
            else:
                self.printEntries(entries)

    def parsePhrase(self, phrase):
        '''returns a list of tuples of the form (filename, linenum, startword)
        where the entire phrase appears, in order, starting at startword in 
        line linenum in file filename'''
        words = phrase.split()
        entries = [self.getEntries(word) for word in words]
        [self.printEntries(e) for e in entries]
        filelist = entries[0].keys()
        phraseStarts = []
        for i in range(1,len(entries)):
            filelist = [fn for fn in filelist if fn in entries[i].keys()]
        for fn in filelist:
            lines = entries[0][fn].keys()
            for i in range(1,len(words)):
                lines = [l for l in lines if l in entries[i][fn].keys()]
            for l in lines:
                for num in entries[0][fn][l]:
                    start = int(num)
                    phraseFound = True
                    for i in range(1,len(words)):
                        if str(start + i) not in entries[i][fn][l]:
                            phraseFound = False
                            break
                    if phraseFound:
                        phraseStarts.append((fn, l, start))
        return phraseStarts

if __name__ == '__main__':
    try:
        filename = argv[1]
    except:
        print "use: python query.py indexfile.txt"
        exit()
    qh = QueryHandler(filename)
#    qh.readQueries()
    print qh.parsePhrase('casing cask')

{'fraction_non_alphanumeric': 0.10870545782589085, 'fraction_numerical': 0.0040595399188092015, 'mean_word_length': 1.3921251348435815, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '578818', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1169, 'n_words': 349}