<commit_before>from pattern.en import parse, Text
from langdetect import detect_langs

class ParseStatistic(object):
    properties = {}
    properties['proper_sentences'] = 0
    properties['too_few_words'] = 0
    properties['first_word_is_number'] = 0
    properties['sentence_contains_brackets'] = 0
    properties['sentence_contains_number'] = 0
    properties['sentence_too_many_comma'] = 0
    properties['too_many_short_words'] = 0
    properties['sentence_in_german'] = 0
    properties['sentence_in_french'] = 0
    properties['sentence_in_spanish'] = 0
    properties['sentence_in_italian'] = 0
    properties['sentence_in_dutch'] = 0
    properties['sentence_not_english'] = 0

class TextParser(object):
    """
    This parses a massive string, that contains an entire book. Splits
    it into sentences and either discards the sentence if the pdf->txt
    conversion was faulty already or returns it.

    This is using pattern.en as a analysis library http://www.clips.ua.ac.be/pages/pattern-en#parser
    The tags of words are being analyzed: http://www.clips.ua.ac.be/pages/mbsp-tags
    """
    def __init__(self):
        self.valid_sentences = []
        self.faulty_sentences = []
        self.MIN_WORD_COUNT = 10
        self.statistic = ParseStatistic()

    @staticmethod
    def get_count_tag(sentence, tag):
        count = 0
        for word in sentence:
            if word.type == tag:
                count += 1
        return count

    @staticmethod
    def contains_tag(sentence, tag):
        for word in sentence:
            if word.type == tag:
                return True

        return False

    @staticmethod
    def get_perc_single_char_words(sentence):
        short_word = 0
        long_word = 0
        for word in sentence:
            if len(word.string) < 2:
                short_word += 1
            else:
                long_word += 1
        perc = float(short_word) / float(short_word + long_word)
        return perc

    def parse(self, text):
        self.valid_sentences = []
        self.faulty_sentences = []

        self.statistic.properties['proper_sentences'] = 0
        self.statistic.properties['too_few_words'] = 0
        self.statistic.properties['first_word_is_number'] = 0
        self.statistic.properties['sentence_contains_brackets'] = 0
        self.statistic.properties['sentence_contains_number'] = 0
        self.statistic.properties['sentence_too_many_comma'] = 0
        self.statistic.properties['too_many_short_words'] = 0
        self.statistic.properties['sentence_in_german'] = 0
        self.statistic.properties['sentence_in_french'] = 0
        self.statistic.properties['sentence_in_spanish'] = 0
        self.statistic.properties['sentence_in_italian'] = 0
        self.statistic.properties['sentence_in_dutch'] = 0
        self.statistic.properties['sentence_not_english'] = 0

        text = Text(parse(text,
                          tokenize=True,
                          tags=True,
                          chunks=True,
                          relations=False,
                          lemmata=False,
                          encoding='utf-8',
                          tagset=None))

        for sentence in text:
            if len(sentence.words) > self.MIN_WORD_COUNT:
                first_word = sentence.words[0]
                first_word_tag = first_word.type
                if self.get_perc_single_char_words(sentence) < 0.5:
                    if first_word_tag != "CD":
                        if not self.contains_tag(sentence, "(") \
                                and not self.contains_tag(sentence, ")") \
                                and not self.contains_tag(sentence, "\""):
                            if self.get_count_tag(sentence, "CD") < 1:
                                if self.get_count_tag(sentence, ",") < 3:
                                    detected_language = str(detect_langs(sentence.string)[0]).split(':')[0]
                                    if detected_language in 'en':
                                        self.valid_sentences.append(sentence)
                                    elif detected_language in 'de':
                                        self.statistic.properties['sentence_in_german'] += 1
                                        self.faulty_sentences.append(sentence)
                                    elif detected_language in 'fr':
                                        self.statistic.properties['sentence_in_french'] += 1
                                        self.faulty_sentences.append(sentence)
                                    elif detected_language in 'es':
                                        self.statistic.properties['sentence_in_spanish'] += 1
                                        self.faulty_sentences.append(sentence)
                                    elif detected_language in 'nl':
                                        self.statistic.properties['sentence_in_dutch'] += 1
                                        self.faulty_sentences.append(sentence)
                                    elif detected_language in 'it':
                                        self.statistic.properties['sentence_in_italian'] += 1
                                        self.faulty_sentences.append(sentence)
                                    else:
                                        self.statistic.properties['sentence_not_english'] += 1
                                        self.faulty_sentences.append(sentence)
                                else:
                                    # the sentence has more than 2 occurrences of a comma(,)
                                    self.statistic.properties['sentence_too_many_comma'] += 1
                                    self.faulty_sentences.append(sentence)
                            else:
                                # the sentence contains a number
                                self.statistic.properties['sentence_contains_number'] += 1
                                self.faulty_sentences.append(sentence)
                        else:
                            # the sentence contains either ( ) or \
                            self.statistic.properties['sentence_contains_brackets'] += 1
                            self.faulty_sentences.append(sentence)
                    else:
                        # first word of the sentence is a number<commit_msg>pdf2text: catches exception of langdetect<commit_after>from pattern.en import parse, Text
import langdetect

class ParseStatistic(object):
    properties = {}
    properties['proper_sentences'] = 0
    properties['too_few_words'] = 0
    properties['first_word_is_number'] = 0
    properties['sentence_contains_brackets'] = 0
    properties['sentence_contains_number'] = 0
    properties['sentence_too_many_comma'] = 0
    properties['too_many_short_words'] = 0
    properties['sentence_in_german'] = 0
    properties['sentence_in_french'] = 0
    properties['sentence_in_spanish'] = 0
    properties['sentence_in_italian'] = 0
    properties['sentence_in_dutch'] = 0
    properties['sentence_not_english'] = 0

class TextParser(object):
    """
    This parses a massive string, that contains an entire book. Splits
    it into sentences and either discards the sentence if the pdf->txt
    conversion was faulty already or returns it.

    This is using pattern.en as a analysis library http://www.clips.ua.ac.be/pages/pattern-en#parser
    The tags of words are being analyzed: http://www.clips.ua.ac.be/pages/mbsp-tags
    """
    def __init__(self):
        self.valid_sentences = []
        self.faulty_sentences = []
        self.MIN_WORD_COUNT = 10
        self.statistic = ParseStatistic()

    @staticmethod
    def get_count_tag(sentence, tag):
        count = 0
        for word in sentence:
            if word.type == tag:
                count += 1
        return count

    @staticmethod
    def contains_tag(sentence, tag):
        for word in sentence:
            if word.type == tag:
                return True

        return False

    @staticmethod
    def get_perc_single_char_words(sentence):
        short_word = 0
        long_word = 0
        for word in sentence:
            if len(word.string) < 2:
                short_word += 1
            else:
                long_word += 1
        perc = float(short_word) / float(short_word + long_word)
        return perc

    def parse(self, text):
        self.valid_sentences = []
        self.faulty_sentences = []

        self.statistic.properties['proper_sentences'] = 0
        self.statistic.properties['too_few_words'] = 0
        self.statistic.properties['first_word_is_number'] = 0
        self.statistic.properties['sentence_contains_brackets'] = 0
        self.statistic.properties['sentence_contains_number'] = 0
        self.statistic.properties['sentence_too_many_comma'] = 0
        self.statistic.properties['too_many_short_words'] = 0
        self.statistic.properties['sentence_in_german'] = 0
        self.statistic.properties['sentence_in_french'] = 0
        self.statistic.properties['sentence_in_spanish'] = 0
        self.statistic.properties['sentence_in_italian'] = 0
        self.statistic.properties['sentence_in_dutch'] = 0
        self.statistic.properties['sentence_not_english'] = 0

        text = Text(parse(text,
                          tokenize=True,
                          tags=True,
                          chunks=True,
                          relations=False,
                          lemmata=False,
                          encoding='utf-8',
                          tagset=None))

        for sentence in text:
            if len(sentence.words) > self.MIN_WORD_COUNT:
                first_word = sentence.words[0]
                first_word_tag = first_word.type
                if self.get_perc_single_char_words(sentence) < 0.5:
                    if first_word_tag != "CD":
                        if not self.contains_tag(sentence, "(") \
                                and not self.contains_tag(sentence, ")") \
                                and not self.contains_tag(sentence, "\""):
                            if self.get_count_tag(sentence, "CD") < 1:
                                if self.get_count_tag(sentence, ",") < 3:
                                    try:
                                        detected_language = str(langdetect.detect_langs(sentence.string)[0]).split(':')[0]
                                        if detected_language in 'en':
                                            self.valid_sentences.append(sentence)
                                        elif detected_language in 'de':
                                            self.statistic.properties['sentence_in_german'] += 1
                                            self.faulty_sentences.append(sentence)
                                        elif detected_language in 'fr':
                                            self.statistic.properties['sentence_in_french'] += 1
                                            self.faulty_sentences.append(sentence)
                                        elif detected_language in 'es':
                                            self.statistic.properties['sentence_in_spanish'] += 1
                                            self.faulty_sentences.append(sentence)
                                        elif detected_language in 'nl':
                                            self.statistic.properties['sentence_in_dutch'] += 1
                                            self.faulty_sentences.append(sentence)
                                        elif detected_language in 'it':
                                            self.statistic.properties['sentence_in_italian'] += 1
                                            self.faulty_sentences.append(sentence)
                                        else:
                                            self.statistic.properties['sentence_not_english'] += 1
                                            self.faulty_sentences.append(sentence)
                                    except langdetect.lang_detect_exception.LangDetectException:
                                        self.statistic.properties['sentence_not_english'] += 1
                                        self.faulty_sentences.append(sentence)
                                else:
                                    # the sentence has more than 2 occurrences of a comma(,)
                                    self.statistic.properties['sentence_too_many_comma'] += 1
                                    self.faulty_sentences.append(sentence)
                            else:
                                # the sentence contains a number
                                self.statistic.properties['sentence_contains_number'] += 1
                                self.faulty_sentences.append(sentence)
                        else:
                            # the sentence contains either ( ) or \
                            self.statistic.properties['sentence_contains_brackets'] += 1
                            self.faulty_sentences.append(sentence)
                    else:
                        # first word of the sentence is a number
{'fraction_non_alphanumeric': 0.08095703859754722, 'fraction_numerical': 0.008125799413136708, 'mean_word_length': 1.4119034657956813, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 86, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3847947', 'n_tokens_mistral': 3409, 'n_tokens_neox': 3091, 'n_words': 629}