<commit_before>#!/usr/bin/env python

import sys
import shelve

# Open just for read
dbpediadb = shelve.open('dbpedia_types.dbm', flag='r')

for line in sys.stdin:
    # remove leading and trailing whitespace
    line = line.strip()
    # split the line into words
    ngram, num = line.split('\t')

    words = ngram.split()
    dbpedia_words = [(i, word) for i, word in enumerate(words) if word[0] == '<' and word[-1] == '>']
    dbp_dict = {}
    for i, word in dbpedia_words:
        if word in dbpediadb:
            dbp_dict[i] = dbpediadb[word]

    if not dbp_dict:
        continue

    ngrams = [words[:]]
    for i, types in dbp_dict.items():
        new_ngrams = []
        for words in ngrams:
            for dbp_type in types:
                new_words = words[:]
                new_words[i] = dbp_type
                new_ngrams.append(new_words)
        ngrams = new_ngrams

    for new_words in ngrams:
        print '%s\t%s' % (' '.join(new_words), num)

dbpediadb.close()<commit_msg>update type parsing<commit_after>#!/usr/bin/env python

import sys
import shelve

# Open just for read
dbpediadb = shelve.open('dbpedia_types.dbm', flag='r')

for line in sys.stdin:
    # remove leading and trailing whitespace
    line = line.strip()
    # split the line into words
    ngram, num = line.split('\t')

    words = ngram.split()
    dbpedia_words = [(i, word) for i, word in enumerate(words) if word[0] == '<' and word[-1] == '>']
    dbp_dict = {}
    for i, word in dbpedia_words:
        if word in dbpediadb:
            dbp_dict[i] = dbpediadb[word]
        else:
            words[i] = words[i][1:-1]

    if not dbp_dict:
        continue

    ngrams = [words[:]]
    for i, types in dbp_dict.items():
        new_ngrams = []
        for words in ngrams:
            for dbp_type in types:
                new_words = words[:]
                new_words[i] = dbp_type
                new_ngrams.append(new_words)
                # take only the first type for now!!
                # TODO: to type or not to type. That is the question.
                break
        ngrams = new_ngrams

    for new_words in ngrams:
        print '%s\t%s' % (' '.join(new_words), num)

dbpediadb.close()
{'fraction_non_alphanumeric': 0.11714156490275893, 'fraction_numerical': 0.0027137042062415195, 'mean_word_length': 2.0937062937062936, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '238158', 'n_tokens_mistral': 787, 'n_tokens_neox': 750, 'n_words': 211}