<commit_before>from ..model import Entity, Name


import logging

log = logging.getLogger()

class BuildDbpediaLexicalisations(object):
    "Build entity and name models from DBpedia Spotlight lexicalisations file."
    ENTITY_SCORE = u'http://dbpedia.org/spotlight/score#uriProbability'
    ENTITY_PRE = u'http://dbpedia.org/resource/'
    NAME_SCORE = u'http://dbpedia.org/spotlight/score#uriGivenSf'
    NAME_PRE = u'http://dbepdia.org/spotlight/id/'

    def __init__(self, inpath, redirect_model_path, name_outpath, entity_outpath):
        self.in_path = inpath
        self.redirect_model_path = redirect_model_path
        self.name_out_path = name_outpath
        self.entity_out_path = entity_outpath

        log.info('Loading redirect model: %s ...', self.redirect_model_path)
        self.redirect_model = marshal.load(open(self.redirect_model_path, 'rb'))

    @classmethod
    def add_arguments(cls, p):
        p.add_argument('inpath', metavar='INFILE')
        p.add_argument('redirect_model_path', metavar='REDIRECT_MODEL_PATH')
        p.add_argument('name_outpath', metavar='NAME_MODEL_OUTFILE')
        p.add_argument('entity_outpath', metavar='ENTITY_MODEL_OUTFILE')
        p.set_defaults(cls=cls)<commit_msg>Fixed import error for dbpedia derived models<commit_after>from ...model import model

import logging
log = logging.getLogger()

class BuildDbpediaLexicalisations(object):
    "Build entity and name models from DBpedia Spotlight lexicalisations file."
    ENTITY_SCORE = u'http://dbpedia.org/spotlight/score#uriProbability'
    ENTITY_PRE = u'http://dbpedia.org/resource/'
    NAME_SCORE = u'http://dbpedia.org/spotlight/score#uriGivenSf'
    NAME_PRE = u'http://dbepdia.org/spotlight/id/'

    def __init__(self, inpath, redirect_model_path, name_outpath, entity_outpath):
        self.in_path = inpath
        self.redirect_model_path = redirect_model_path
        self.name_out_path = name_outpath
        self.entity_out_path = entity_outpath

        log.info('Loading redirect model: %s ...', self.redirect_model_path)
        self.redirect_model = marshal.load(open(self.redirect_model_path, 'rb'))

    @classmethod
    def add_arguments(cls, p):
        p.add_argument('inpath', metavar='INFILE')
        p.add_argument('redirect_model_path', metavar='REDIRECT_MODEL_PATH')
        p.add_argument('name_outpath', metavar='NAME_MODEL_OUTFILE')
        p.add_argument('entity_outpath', metavar='ENTITY_MODEL_OUTFILE')
        p.set_defaults(cls=cls)
{'fraction_non_alphanumeric': 0.11093247588424437, 'fraction_numerical': 0.0, 'mean_word_length': 4.069246435845214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7459450', 'n_tokens_mistral': 845, 'n_tokens_neox': 816, 'n_words': 132}