<commit_before>import threading
import Queue
import logging

WAIT_QUEUE_TIMEOUT = 2 # in seconds

class WikipediaPreProcessor(threading.Thread):
    '''The WikipediaPreProcessor class preprocesses text and removes tags, unnecessary links and
       other information and keeps the text only
    '''

    '''constructor

       @param input_queue if the preprocessor is used as a thread, articles are read from this queue [optional]
       @param output_queue if the preprocessor is used as a thread, articles are written to this queue [optional]
    '''
    def __init__(self, input_queue=None, output_queue=None):
        threading.Thread.__init__(self)
        self._input_queue = input_queue
        self._output_queue = output_queue
        self._end = False

        self._REMOVE_PAIRS=(
            (u'{{', u'}}'),
            (u'=====', u'====='),
            (u'====', u'===='),
            (u'===', u'==='),
            (u'==', u'=='),
            (u'[http', u']'),
            (u'[File:', u']'),
            (u'[Category:', u']')
        )
        # html tags can end with /> or </tag-name> and needs to be handled separately
        # <!-- --> comments also as they can spread multiple lines

    '''processes a single article and cleans the text

       @param article a dictionary with a text field that contains the text (will be modified to hold the new text)

       @return the article with clean text only containing valid links
    '''
    def process(self, article):
        new_text = ''

        next_line_starts_in_comment = False
        line_starts_in_comment = False
        next_line_in_tag = 0
        line_in_tag = 0
        html_tags = []

        # iterate over lines
        lines = article['text'].strip().split('\n')
        for line in lines:
            line = line.strip()
            # reset html tags if an empty line (probably one was not properly closed somewhere)
            if len(line) == 0:
                html_tags = []

            # STEP 1 - remove hyphens and restore tags
            line = line.replace("'''", "")
            line = line.replace("''", "")
            line = line.replace('&lt;', '<')
            line = line.replace('&gt;', '>')

            # keep <sub> and <sup> tags if preceeded by uppercase letter (chemical element)
            index = 1
            while line.find('<sub>', index) != -1:
                index = line.find('<sub>', index)+5
                letter_before = line[index-6]
                end = line.find('</sub>', index)
                content = line[index:end]
                # check if content is numeric and letter is uppercase
                if content.isdigit() and letter_before == letter_before.upper():
                    line = line[:index-5] + content + line[end+6:]
                    index = index-5

            # check if the line starts in a comment
            line_starts_in_comment = next_line_starts_in_comment
            next_line_starts_in_comment = False

            # check if the line starts in a tag
            line_in_tag = next_line_in_tag
            next_line_in_tag = 0

            # STEP 2 - remove comments
            while line.find('<!--') != -1 or line_starts_in_comment:
                start = 0
                if not line_starts_in_comment:
                    start = line.find('<!--')
                line_starts_in_comment = False
                end = line.find('-->')
                if end == -1:
                    next_line_starts_in_comment = True
                    line = line[0:start]
                else:
                    # a problem occurred, just ignore the line
                    if start > end:
                        line = ''
                    else:
                        line = line[0:start] + line[end+3:]

            # STEP 3 - remove html tags
            index = 0
            outer_start_tag = line.find('<')
            # if the line already starts within an html tag
            if len(html_tags) != 0:
                outer_start_tag = 0
            while line.find('<', index) != -1:
                start = False
                index = line.find('<', index)+1
                # if the tag is the last sign in the line, just remove it
                if index == len(line):
                    line = line[:-1]
                else:
                    end_tag = line.find('>', index)
                    # if tag is a closing tag
                    if line[index] == '/':
                        # this query is necessary as some invalid close tags appear on wikipedia - nothging to be done about that
                        if len(html_tags) != 0:
                            html_tags.pop()
                        # this is the outermost html tag
                        if len(html_tags) == 0:
                            line = line[:outer_start_tag] + line[end_tag+1:]
                            # start with next tag
                            outer_start_tag = line.find('<')
                            index = 0
                    # not a closing tag
                    else:
                        # a simple tag without an ending one, just remove it
                        if line[end_tag-1] == '/':
                            line = line[0:index-1] + line[end_tag+1:]
                            index-= 1
                            # if this was the outermost tag, start from the next tag
                            if index == outer_start_tag:
                                outer_start_tag = line.find('<')
                        # a normal tag is simply pushed to the stack
                        else:
                            tag_name = line[index:end_tag]
                            # ignore unclean br tags
                            if tag_name != 'br':
                                html_tags.append(line[index:end_tag])

            # TODO: refactor
            if len(html_tags) > 0:
                # there is an opening tag somewhere
                if line.find('<') != -1:
                    line = line[:line.find('<')]
                else: # everything is already within a tag
                    line = ''

            # STEP 4 - remove invalid lines
            # simply ignore too short lines and those that start with an incorrect token
            if len(line) > 4 and line[0:2] != ' |' and line[0] != '|' and line[0] != '!':

                # STEP 5 - remove incorrect links
                line = self._remove_incorrect_links(line)

                # STEP 6 - remove pairs
                for pair in self._REMOVE_PAIRS:<commit_msg>fixed minor bug for invalid html tag<commit_after>import threading
import Queue
import logging

WAIT_QUEUE_TIMEOUT = 2 # in seconds

class WikipediaPreProcessor(threading.Thread):
    '''The WikipediaPreProcessor class preprocesses text and removes tags, unnecessary links and
       other information and keeps the text only
    '''

    '''constructor

       @param input_queue if the preprocessor is used as a thread, articles are read from this queue [optional]
       @param output_queue if the preprocessor is used as a thread, articles are written to this queue [optional]
    '''
    def __init__(self, input_queue=None, output_queue=None):
        threading.Thread.__init__(self)
        self._input_queue = input_queue
        self._output_queue = output_queue
        self._end = False

        self._REMOVE_PAIRS=(
            (u'{{', u'}}'),
            (u'=====', u'====='),
            (u'====', u'===='),
            (u'===', u'==='),
            (u'==', u'=='),
            (u'[http', u']'),
            (u'[File:', u']'),
            (u'[Category:', u']')
        )
        # html tags can end with /> or </tag-name> and needs to be handled separately
        # <!-- --> comments also as they can spread multiple lines

    '''processes a single article and cleans the text

       @param article a dictionary with a text field that contains the text (will be modified to hold the new text)

       @return the article with clean text only containing valid links
    '''
    def process(self, article):
        new_text = ''

        next_line_starts_in_comment = False
        line_starts_in_comment = False
        next_line_in_tag = 0
        line_in_tag = 0
        html_tags = []

        # iterate over lines
        lines = article['text'].strip().split('\n')
        for line in lines:
            line = line.strip()
            # reset html tags if an empty line (probably one was not properly closed somewhere)
            if len(line) == 0:
                html_tags = []

            # STEP 1 - remove hyphens and restore tags
            line = line.replace("'''", "")
            line = line.replace("''", "")
            line = line.replace('&lt;', '<')
            line = line.replace('&gt;', '>')

            # keep <sub> and <sup> tags if preceeded by uppercase letter (chemical element)
            index = 1
            while line.find('<sub>', index) != -1:
                index = line.find('<sub>', index)+5
                letter_before = line[index-6]
                end = line.find('</sub>', index)
                content = line[index:end]
                # check if content is numeric and letter is uppercase
                if content.isdigit() and letter_before == letter_before.upper():
                    line = line[:index-5] + content + line[end+6:]
                    index = index-5

            # check if the line starts in a comment
            line_starts_in_comment = next_line_starts_in_comment
            next_line_starts_in_comment = False

            # check if the line starts in a tag
            line_in_tag = next_line_in_tag
            next_line_in_tag = 0

            # STEP 2 - remove comments
            while line.find('<!--') != -1 or line_starts_in_comment:
                start = 0
                if not line_starts_in_comment:
                    start = line.find('<!--')
                line_starts_in_comment = False
                end = line.find('-->')
                if end == -1:
                    next_line_starts_in_comment = True
                    line = line[0:start]
                else:
                    # a problem occurred, just ignore the line
                    if start > end:
                        line = ''
                    else:
                        line = line[0:start] + line[end+3:]

            # STEP 3 - remove html tags
            index = 0
            outer_start_tag = line.find('<')
            # if the line already starts within an html tag
            if len(html_tags) != 0:
                outer_start_tag = 0
            while line.find('<', index) != -1:
                start = False
                index = line.find('<', index)+1
                # if the tag is the last sign in the line, just remove it
                if index == len(line):
                    line = line[:-1]
                else:
                    end_tag = line.find('>', index)
                    if end_tag == -1:
                        line = line[:line.find('<', index)]
                    else:
                        # if tag is a closing tag
                        if line[index] == '/':
                            # this query is necessary as some invalid close tags appear on wikipedia - nothging to be done about that
                            if len(html_tags) != 0:
                                html_tags.pop()
                            # this is the outermost html tag
                            if len(html_tags) == 0:
                                line = line[:outer_start_tag] + line[end_tag+1:]
                                # start with next tag
                                outer_start_tag = line.find('<')
                                index = 0
                        # not a closing tag
                        else:
                            # a simple tag without an ending one, just remove it
                            if line[end_tag-1] == '/':
                                line = line[0:index-1] + line[end_tag+1:]
                                index-= 1
                                # if this was the outermost tag, start from the next tag
                                if index == outer_start_tag:
                                    outer_start_tag = line.find('<')
                            # a normal tag is simply pushed to the stack
                            else:
                                tag_name = line[index:end_tag]
                                # ignore unclean br tags
                                if tag_name != 'br':
                                    html_tags.append(line[index:end_tag])

            # TODO: refactor
            if len(html_tags) > 0:
                # there is an opening tag somewhere
                if line.find('<') != -1:
                    line = line[:line.find('<')]
                else: # everything is already within a tag
                    line = ''

            # STEP 4 - remove invalid lines
            # simply ignore too short lines and those that start with an incorrect token
            if len(line) > 4 and line[0:2] != ' |' and line[0] != '|' and line[0] != '!':

                # STEP 5 - remove incorrect links
                line = self._remove_incorrect_links(line)

                # STEP 6 - remove pairs
                for pair in self._REMOVE_PAIRS:
{'fraction_non_alphanumeric': 0.09694980982921918, 'fraction_numerical': 0.006935640241628757, 'mean_word_length': 1.306501547987616, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2124267', 'n_tokens_mistral': 3513, 'n_tokens_neox': 3299, 'n_words': 1116}