<commit_before># -*- coding: utf-8 -*-
from terminators import HEADERS, FOOTERS
import logging


def get_metadata(lines, key, sep=':'):
    value = None
    for line in lines:
        if line.startswith(key + sep):
            value = line.split(sep)[1].strip()
            break
    if value is None:
        logging.warning('no %s found' % key)
    return value


def strip_headers(lines):
    """Remove lines that are part of the Project Gutenberg header or footer.

    This is a port of the C++ version by <NAME>
    Link: http://www14.in.tum.de/spp1307/src/strip_headers.cpp

    Args:
        lines (list): the lines of text to filter

    Returns:
        list: the lines of text that don't belong to a header or footer

    """
    out = []
    i = 0
    reset = True
    footer_found = False

    for line in lines:
        if len(line) <= 12:
            continue  # just a shortcut for short lines

        reset = False

        if i <= 600:
            # Check if the header ends here
            if any(line.startswith(header) for header in HEADERS):
                reset = True

            # If it's the end of the header, delete the output produced so far.
            # May be done several times, if multiple lines occur indicating the
            # end of the header
            if reset:
                out = []
                continue

        if i >= 100:
            # Check if the footer begins here
            if any(line.startswith(footer) for footer in FOOTERS):
                footer_found = True

            # If it's the beginning of the footer, stop output
            if footer_found:
                break

        out.append(line)
        i += 1

    return out


if __name__ == '__main__':
    import argparse
    import sys

    doc = 'Removes Project Gutenberg headers and footers.'
    parser = argparse.ArgumentParser(description=doc)
    parser.add_argument('infile', nargs='?', type=argparse.FileType('r'),
                        default=sys.stdin, help='gutenberg text to clean-up')
    parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'),
                        default=sys.stdout, help='cleaned text goes here')
    args = parser.parse_args()

    for line in strip_headers(args.infile):
        args.outfile.write(line)
<commit_msg>Reduce noise - strip whitespace from etext lines<commit_after># -*- coding: utf-8 -*-
from terminators import HEADERS, FOOTERS
import logging


def get_metadata(lines, key, sep=':'):
    value = None
    for line in lines:
        if line.startswith(key + sep):
            value = line.split(sep)[1].strip()
            break
    if value is None:
        logging.warning('no %s found' % key)
    return value


def strip_headers(lines):
    """Remove lines that are part of the Project Gutenberg header or footer.

    This is a port of the C++ version by <NAME>
    Link: http://www14.in.tum.de/spp1307/src/strip_headers.cpp

    Args:
        lines (list): the lines of text to filter

    Returns:
        list: the lines of text that don't belong to a header or footer

    """
    out = []
    i = 0
    reset = True
    footer_found = False

    for line in lines:
        if len(line) <= 12:
            continue  # just a shortcut for short lines

        reset = False

        if i <= 600:
            # Check if the header ends here
            if any(line.startswith(header) for header in HEADERS):
                reset = True

            # If it's the end of the header, delete the output produced so far.
            # May be done several times, if multiple lines occur indicating the
            # end of the header
            if reset:
                out = []
                continue

        if i >= 100:
            # Check if the footer begins here
            if any(line.startswith(footer) for footer in FOOTERS):
                footer_found = True

            # If it's the beginning of the footer, stop output
            if footer_found:
                break

        out.append(line.strip())
        i += 1

    return out


if __name__ == '__main__':
    import argparse
    import sys

    doc = 'Removes Project Gutenberg headers and footers.'
    parser = argparse.ArgumentParser(description=doc)
    parser.add_argument('infile', nargs='?', type=argparse.FileType('r'),
                        default=sys.stdin, help='gutenberg text to clean-up')
    parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'),
                        default=sys.stdout, help='cleaned text goes here')
    args = parser.parse_args()

    for line in strip_headers(args.infile):
        args.outfile.write(line)

{'fraction_non_alphanumeric': 0.09073317566114814, 'fraction_numerical': 0.0077402709094818315, 'mean_word_length': 1.9368686868686869, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6472310', 'n_tokens_mistral': 1375, 'n_tokens_neox': 1266, 'n_words': 466}