<commit_before>##by: <NAME>
##date updated: 3/11/11
##updated by: <NAME>
## revised 5/2014 by <NAME>

# import functionality from utility module
from util import *

def classify_line(line):
    trial_types = {
    '2' : 's',
    '6' : 'q',
    '7' : 'q',
    }
    thing = line.split()[4]
    if thing in trial_types:
        return trial_types[thing]
    return 'reject'


def parse_da1_file(file_name):
    subj_number = get_subj_num(file_name)
    with open(file_name) as da1file:
        non_empty = [line for line in da1file if line]
        sentences = [line for line in non_empty if classify_line(line) is 's']
        questions = [line for line in non_empty if classify_line(line) is 'q']
        rejects = [line for line in non_empty if classify_line(line) is 'reject']
    return (subj_number, sentences, questions, rejects)

<commit_msg>cleaned up classify_line<commit_after>##by: <NAME>
##date updated: 3/11/11
##updated by: <NAME>
## revised 5/2014 by <NAME>

# import functionality from utility module
from util import *

def classify_line(line):
    '''Given a line (as a string), determines what type of trial this line is:
    whether it is a question, a sentence or a rejected trial.
    '''
    trial_types = {
    '2' : 's',
    '6' : 'q',
    '7' : 'q',
    }
    item_type = line.split()[4]
    if item_type in trial_types:
        return trial_types[item_type]
    return 'reject'


def parse_da1_file(file_name):
    subj_number = get_subj_num(file_name)
    with open(file_name) as da1file:
        non_empty = [line for line in da1file if line]
        sentences = [line for line in non_empty if classify_line(line) is 's']
        questions = [line for line in non_empty if classify_line(line) is 'q']
        rejects = [line for line in non_empty if classify_line(line) is 'reject']
    return (subj_number, sentences, questions, rejects)


{'fraction_non_alphanumeric': 0.11134903640256959, 'fraction_numerical': 0.018201284796573874, 'mean_word_length': 3.267123287671233, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1301422', 'n_tokens_mistral': 650, 'n_tokens_neox': 620, 'n_words': 207}