<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Clean the provided txt file before passing it to the indexer

./tidy.py issue.txt > clean.txt
"""
import fileinput

buf = ''


# do the stuff
for line in fileinput.input():
    line = line.strip()

    # empty headings
    if line.startswith('\x1f\x1d\x0b'):
        pass
    # page numbers
    elif line.startswith('\x0c'):
        continue
    # page breaks - headings
    elif line.startswith('\x1f\x1d'):
        line = line[2:]  # remove magic characters

        # ignore empty headings followed by the title
        if line == '':
            continue

        line = "\n{}".format(line)

    # headings for subsequent pages in the same chapter - can be skipped
    if line.startswith('\x0c'):
        continue

    # merge consecutive lines that end with a line break (-)
    if line.endswith('-') and not line.endswith(' -'):
        buf += line[:-1]
    # merge lines that belong to a single sentence
    elif not line.endswith('.'):
        buf += line + ' '
    else:
        print (buf + line).rstrip()
        buf = ''

# print any remaining lines
if buf != '':
    print buf<commit_msg>tidy.py: refactoring<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Clean the provided txt file before passing it to the indexer

./tidy.py issue.txt > clean.txt
"""
import fileinput
import sys


class TextTidy(object):
    def __init__(self, _in):
        self._in = _in

    def tidy(self, output=sys.stdout):
        buf = ''

        # do the stuff
        for line in self._in:
            line = line.strip()

            # empty headings
            if line.startswith('\x1f\x1d\x0b'):
                pass
            # page numbers
            elif line.startswith('\x0c'):
                continue
            # page breaks - headings
            elif line.startswith('\x1f\x1d'):
                line = line[2:]  # remove magic characters

                # ignore empty headings followed by the title
                if line == '':
                    continue

                line = "\n{}".format(line)

            # headings for subsequent pages in the same chapter - can be skipped
            if line.startswith('\x0c'):
                continue

            # merge consecutive lines that end with a line break (-)
            if line.endswith('-') and not line.endswith(' -'):
                buf += line[:-1]
            # merge lines that belong to a single sentence
            elif not line.endswith('.'):
                buf += line + ' '
            else:
                output.write((buf + line).rstrip() + "\n")
                buf = ''

        # print any remaining lines
        if buf != '':
            output.write(buf + "\n")


def main():
    tidy = TextTidy(fileinput.input())
    tidy.tidy(sys.stdout)

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.12190010478519037, 'fraction_numerical': 0.006985679357317499, 'mean_word_length': 1.7121212121212122, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5943107', 'n_tokens_mistral': 858, 'n_tokens_neox': 824, 'n_words': 257}