<commit_before>from nltk import word_tokenize, pos_tag, RegexpParser, Tree, data, sent_tokenize
from nltk.corpus import wordnet


class Word(object):
    """Understands the atomic composition of letters"""
    def __init__(self, text, tag=None):
        if type(text) == list or type(text) == tuple and tag is None:
            tag = text[1]
            text = text[0]
        if tag is None: tag = pos_tag([text])[0][1]
        self.text = unicode(text)
        self.tag = str(tag)

    def __eq__(self, other):
        return isinstance(other, self.__class__) and \
            unicode.lower(self.text) == unicode.lower(other.text)

    def __str__(self):
        return self.text

    def is_type(self, *features):
        if features[0].__class__ is tuple: features = [e for tupl in features for e in tupl]
        return False not in [self.feature_set()[f] for f in features]

    def is_synonym_of(self, compar):
        if compar.__class__ is Word: compar = compar.text
        return self.text in set(w for l in wordnet.synsets(compar) for w in l.lemma_names())

    def feature_set(self):
        return {
            'verb': self.tag[0] == 'V',
            'noun': self.tag[0] == 'N',
            'belief': unicode.lower(self.text) in BELIEF_WORDS,
            'attitude': unicode.lower(self.text) in ATTITUDE_WORDS,
            'being': unicode.lower(self.text) in BEING_WORDS,
            'nonverb': not self.tag[0] == 'V',
            'future_tense': self.tag in ["MD"],
            'present_tense': self.tag in ["VBP", "VBZ","VBG"],
            'past_tense': self.tag in ["VBD", "VBN"]
        }

    def __hash__(self):
        return hash(frozenset(self.text))


class Sentence(object):
    """Understands series of words that forms a complete thought"""
    def __init__(self, text):
        self.text = unicode(text).strip()
        tokenList = word_tokenize(text)
        self.POStags = pos_tag(tokenList)
        self.words = []
        for eachWord in self.POStags:
            text = eachWord[0]
            tag = eachWord[1]
            self.words.append(Word(text, tag))
        self.chunkedSentence = []

    def __eq__(self, other):
        return isinstance(other, self.__class__) and self.text == other.text

    def __hash__(self):
        return hash(frozenset(self.text))

    def __str__(self):
        return self.text
<commit_msg>Fixes two small syntax errors<commit_after>from nltk import word_tokenize, pos_tag, RegexpParser, Tree, sent_tokenize
from nltk.corpus import wordnet


class Word(object):
    """Understands the atomic composition of letters"""
    def __init__(self, text, tag=None):
        if type(text) == list or type(text) == tuple and tag is None:
            tag = text[1]
            text = text[0]
        if tag is None: tag = pos_tag([text])[0][1]
        self.text = unicode(text)
        self.tag = str(tag)

    def __eq__(self, other):
        return isinstance(other, self.__class__) and \
            unicode.lower(self.text) == unicode.lower(other.text)

    def __str__(self):
        return self.text

    def is_type(self, *features):
        if features[0].__class__ is tuple: features = [e for tupl in features for e in tupl]
        return False not in [self.feature_set()[f] for f in features]

    def is_synonym_of(self, compar):
        if compar.__class__ is Word: compar = compar.text
        return self.text in set(w for l in wordnet.synsets(compar) for w in l.lemma_names())

    def feature_set(self):
        return {
            'verb': self.tag[0] == 'V',
            'noun': self.tag[0] == 'N',
            'belief': unicode.lower(self.text) in BELIEF_WORDS,
            'attitude': unicode.lower(self.text) in ATTITUDE_WORDS,
            'being': unicode.lower(self.text) in BEING_WORDS,
            'nonverb': not self.tag[0] == 'V',
            'future_tense': self.tag in ["MD"],
            'present_tense': self.tag in ["VBP", "VBZ", "VBG"],
            'past_tense': self.tag in ["VBD", "VBN"]
        }

    def __hash__(self):
        return hash(frozenset(self.text))


class Sentence(object):
    """Understands series of words that forms a complete thought"""
    def __init__(self, text):
        self.text = unicode(text).strip()
        tokenList = word_tokenize(text)
        self.POStags = pos_tag(tokenList)
        self.words = []
        for eachWord in self.POStags:
            text = eachWord[0]
            tag = eachWord[1]
            self.words.append(Word(text, tag))
        self.chunkedSentence = []

    def __eq__(self, other):
        return isinstance(other, self.__class__) and self.text == other.text

    def __hash__(self):
        return hash(frozenset(self.text))

    def __str__(self):
        return self.text

{'fraction_non_alphanumeric': 0.12428782443553492, 'fraction_numerical': 0.00422029964127453, 'mean_word_length': 2.4174477289113194, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2424655', 'n_tokens_mistral': 1527, 'n_tokens_neox': 1478, 'n_words': 406}