<commit_before>"""
test_data.py

Reads, cleans and returns a list of solved or unsolved sudoku puzzle strings
contained in the file provided

:created on: 20160616
__author__ = '<NAME>'
:License: GPL3
"""


def read_data_from(filename):
    """
    reads the text in file_name
    :rtype: list
    :param filename: The name of the file to read
    :return: the list of the file content
    """
    with open(filename, 'r') as file:
        data = file.readlines()
    return data


def filter_and_clean(raw_data):
    """
    strips white space, tabs and line feeds at end of each line
    omits lines starting with '#'
    :param raw_data:
    :return: a python list of cleaned up data
    """
    clean_data = []
    for line in raw_data:
        line = line.strip(' \t\n\r')
        if line.startswith('#') or len(line) == 0:
            continue
        elif all(char.isdigit() or (char == '.') for char in line):
            clean_data.append(line)
    return clean_data


def get_data_from(filename):
    """
    obtains cleaned up data from filename
    :param filename: name of the file we want to obtain data from
    :return: a python list of cleaned up data [list of strings of digits or '.'
    """
    raw_data = read_data_from(filename)
    clean_data = filter_and_clean(raw_data)
    return clean_data


if __name__ == '__main__':
    filenames = ['solved_10000_grids_startwith_123456789.txt',
                 'unsolved_10000_grids_45_numbers.txt',
                 'unsolved_10000_grids_40_numbers.txt',
                 'unsolved_10000_grids_35_numbers.txt',
                 'unsolved_10000_grids_30_numbers.txt',
                 'unsolved_10000_grids_25_numbers.txt']

    for filename in filenames:
        raw_data = get_data_from('../resources/' + filename)
        clean_data = filter_and_clean(raw_data)
        print(filename)
        for line in clean_data[:10]:
            print(line)
        print()
<commit_msg>Made minor changes to comply to pep8, typos and some weak warnings<commit_after>"""
test_data.py

Reads, cleans and returns a list of solved or unsolved sudoku puzzle strings
contained in the file provided

:created on: 20160616
__author__ = '<NAME>'
:License: GPL3
"""


def read_data_from(filename):
    """
    reads the text in file_name
    :rtype: list
    :param filename: The name of the file to read
    :return: the list of the file content
    """
    with open(filename, 'r') as file:
        data = file.readlines()
    return data


def filter_and_clean(raw_data):
    """
    strips white space, tabs and line feeds at end of each line
    omits lines starting with '#'
    :param raw_data: a list of data that needs clean up
    :return: a python list of cleaned up data
    """
    clean_data = []
    for line in raw_data:
        line = line.strip(' \t\n\r')
        if line.startswith('#') or len(line) == 0:
            continue
        elif all(char.isdigit() or (char == '.') for char in line):
            clean_data.append(line)
    return clean_data


def get_data_from(filename):
    """
    obtains cleaned up data from filename
    :param filename: name of the file we want to obtain data from
    :return: a python list of cleaned up data [list of strings of digits or '.'
    """
    raw_data = read_data_from(filename)
    clean_data = filter_and_clean(raw_data)
    return clean_data


if __name__ == '__main__':
    filenames = ['solved_10000_grids_startwith_123456789.txt',
                 'unsolved_10000_grids_45_numbers.txt',
                 'unsolved_10000_grids_40_numbers.txt',
                 'unsolved_10000_grids_35_numbers.txt',
                 'unsolved_10000_grids_30_numbers.txt',
                 'unsolved_10000_grids_25_numbers.txt']

    for filename in filenames:
        raw_data = get_data_from('../resources/' + filename)
        clean_data = filter_and_clean(raw_data)
        print(filename)
        for line in clean_data[:10]:
            print(line)
        print()
{'fraction_non_alphanumeric': 0.09086332746035741, 'fraction_numerical': 0.03095897306821042, 'mean_word_length': 2.635864592863678, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4849338', 'n_tokens_mistral': 1328, 'n_tokens_neox': 1210, 'n_words': 379}