<commit_before>#! /usr/bin/python

import numpy as np
import os
import re
from sklearn import datasets, cross_validation
from sklearn.base import TransformerMixin
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.metrics import classification_report
from sklearn.naive_bayes import GaussianNB
from sklearn.pipeline import Pipeline

MESSAGES_DIR = "data/messages/"
MESSAGE_FILENAME_FORMAT = "msg-%d.txt"
JUDGMENTS_PATH = "data/judgments/judgments.txt"

def load_messages():
    messages = []

    # Read message files in numeric order. os.listdir() returns them sorted by string, not message
    # number.
    filenames = os.listdir(MESSAGES_DIR)
    num_messages = len(filenames)
    for i in range(num_messages):
        filename = MESSAGE_FILENAME_FORMAT % i
        with open(MESSAGES_DIR + filename) as message_file:
            messages.append(message_file.read())

    return messages

def load_sentence_judgments():
    judgments = []
    with open(JUDGMENTS_PATH) as judgments_file:
        for line in judgments_file:
            judgments.append([int(x) for x in line.split()[2:] if len(line.split()) > 2])

    return judgments

def load_sentences():
    messages = load_messages()
    judgments = load_sentence_judgments()
    action_sentences = []
    no_action_sentences = []
    for i in range(len(messages)):
        message = messages[i]
        sentences = parse_sentences(message)
        action_indices = judgments[i]
        if len(action_indices) > 0:
            for i in range(0, len(action_indices), 2):
                start_index = action_indices[i]
                length = action_indices[i+1]
                stop_index = start_index + length
                action_sentence = message[start_index:stop_index].strip().replace('\n', ' ')
                if action_sentence in sentences:
                    action_sentences.append(action_sentence)
                    sentences.remove(action_sentence)
        no_action_sentences.extend(sentences)

    target = [1 for _ in action_sentences]
    target.extend([0 for _ in no_action_sentences])
    action_sentences.extend(no_action_sentences)
    return action_sentences, target

def parse_sentences(message):
    # Split the sentence on periods, exclamation marks, and double newlines. Recombine punctuation
    # marks with their sentences.
    sentences = reduce(lambda acc, elem: acc[:-1] + [acc[-1] + elem] \
        if elem == '.' or elem == '?' or elem == '!' \
        else acc + [elem], re.split(r'([\.\!\?]|\n\n)', message), [])

    # Strip sentences of extra white space.
    # Replace internal newlines with spaces so that newlines don't trip up sklearn tokenizers.
    # Remove all sentences that have length 0 or are completely comprised of whitespace.
    # Remove any sentence starting with the 'From:' header, which should remove the From:, To:, 
    # and Subject:
    sentences = [s.strip().replace('\n', ' ') for s in sentences if len(s) > 0 and not s.isspace() and not s.startswith('From:')]
<commit_msg>Randomizes order of sentence-level examples.<commit_after>#! /usr/bin/python

import numpy as np
import os
import random
import re
from sklearn import datasets, cross_validation
from sklearn.base import TransformerMixin
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.metrics import classification_report
from sklearn.naive_bayes import GaussianNB
from sklearn.pipeline import Pipeline

MESSAGES_DIR = "data/messages/"
MESSAGE_FILENAME_FORMAT = "msg-%d.txt"
JUDGMENTS_PATH = "data/judgments/judgments.txt"

def load_messages():
    messages = []

    # Read message files in numeric order. os.listdir() returns them sorted by string, not message
    # number.
    filenames = os.listdir(MESSAGES_DIR)
    num_messages = len(filenames)
    for i in range(num_messages):
        filename = MESSAGE_FILENAME_FORMAT % i
        with open(MESSAGES_DIR + filename) as message_file:
            messages.append(message_file.read())

    return messages

def load_sentence_judgments():
    judgments = []
    with open(JUDGMENTS_PATH) as judgments_file:
        for line in judgments_file:
            judgments.append([int(x) for x in line.split()[2:] if len(line.split()) > 2])

    return judgments

def load_sentences():
    messages = load_messages()
    judgments = load_sentence_judgments()
    action_sentences = []
    no_action_sentences = []
    for i in range(len(messages)):
        message = messages[i]
        sentences = parse_sentences(message)
        action_indices = judgments[i]
        if len(action_indices) > 0:
            for i in range(0, len(action_indices), 2):
                start_index = action_indices[i]
                length = action_indices[i+1]
                stop_index = start_index + length
                action_sentence = message[start_index:stop_index].strip().replace('\n', ' ')
                if action_sentence in sentences:
                    action_sentences.append(action_sentence)
                    sentences.remove(action_sentence)
        no_action_sentences.extend(sentences)

    target = [1 for _ in action_sentences]
    target.extend([0 for _ in no_action_sentences])
    sentences = action_sentences
    sentences.extend(no_action_sentences)

    combined = zip(target, sentences)
    random.shuffle(combined)
    target[:], sentences[:] = zip(*combined)
    return sentences, target

def parse_sentences(message):
    # Split the sentence on periods, exclamation marks, and double newlines. Recombine punctuation
    # marks with their sentences.
    sentences = reduce(lambda acc, elem: acc[:-1] + [acc[-1] + elem] \
        if elem == '.' or elem == '?' or elem == '!' \
        else acc + [elem], re.split(r'([\.\!\?]|\n\n)', message), [])

    # Strip sentences of extra white space.
    # Replace internal newlines with spaces so that newlines don't trip up sklearn tokenizers.
    # Remove all sentences that have length 0 or are completely comprised of whitespace.
    # Remove any sentence starting with the 'From:' header, which should remove the From:, To:, 
    # and Subject:
    sentences = [s.strip().replace('\n', ' ') for s in sentences if len(s) > 0 and not s.isspace() and not s.startswith('From:')]

{'fraction_non_alphanumeric': 0.09746496614706346, 'fraction_numerical': 0.003778932451582428, 'mean_word_length': 3.214996682149967, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '734903', 'n_tokens_mistral': 1902, 'n_tokens_neox': 1817, 'n_words': 531}