<commit_before>from __future__ import print_function

import csv
import nltk
import sys
import time

from nltk.classify import NaiveBayesClassifier

def word_feats(words):
    return dict([(word, True) for word in words])

def runSentimentAnalysis(checkContiguousMessages = True, checkConversations = True, checkWeeks = True):
    
    print("Performing sentiment analysis...")
    
    f = open("sentimentTraining")
    tweets = [line.strip() for line in f]
    totalTweets = len(tweets)
    
    features = []
    
    count = 0
    
    # Takes a while.
    for tweet in tweets:
        if count % 10000 == 0:
            percent = 100 * count / totalTweets
            sys.stdout.write("\rTraining {0}% complete...".format(percent))
            sys.stdout.flush()
        [sentiment, sentence] = tweet.split("[SEP]")
        # sentence = unicode(sentence, 'utf-8')
        posOrNeg = None
        if sentiment == "0":
            posOrNeg = "neg"
        elif sentiment == "4":
            posOrNeg = "pos"
        else:
            continue
        tokens = nltk.word_tokenize(sentence)
        feats = word_feats(tokens)
        features.append((feats, posOrNeg))
        count += 1
    
    print()
    classifier = NaiveBayesClassifier.train(features)
    
    classifier.show_most_informative_features(100)
    
    messageSentimentAnalysis(classifier)
    if checkContiguousMessages:
        contiguousMessagesSentimentAnalysis(classifier)
    
    if checkConversations:
        conversationSentimentAnalysis(classifier)
    
    if checkWeeks:
        weeklySentimentAnalysis(classifier)

def messageSentimentAnalysis(classifier):
    print("Getting message sentiment scores...")
    
    conversations = open("Conversations.txt")
    
    fieldNames = ["sentiment", "sender"]
    writer = csv.DictWriter(open("Files/messageSentiments.csv", "w"), fieldNames)
    writer.writeheader()
    
    sentimentScores = []
    
    # Individual messages.
    for line in conversations:
        contents = line.strip().split("[SEP]")
        sender = contents[1].replace(",", "")
        message = " ".join(contents[2:])
        tokens = nltk.word_tokenize(message)
        feats = word_feats(tokens)
        probs = classifier.prob_classify(feats)
        pos = probs.prob("pos")
        row = {"sentiment": pos, "sender": sender}
        writer.writerow(row)
        sentimentScores.append((pos, message))
    
    sentimentScores.sort(key = lambda message: message[0], reverse = True)
    
    output = open("Files/messageSentimentScores", "w")
    for message in sentimentScores:
        print("{0}: {1}".format(message[0], message[1]), file = output)
    
    output.close()

def contiguousMessagesSentimentAnalysis(classifier):
    print("Getting contiguous messages sentiment scores...")
    
    conversations = open("Conversations.txt")
    
    fieldNames = ["sentiment", "sender"]
    writer = csv.DictWriter(open("Files/contiguousMessageSentiments.csv", "w"), fieldNames)
    writer.writeheader()
    
    sentimentScores = []
    currentSender = None
    currentMessage = ""
    currentTokens = []
    pattern = "%Y-%m-%d %H:%M:%S"
    gapTime = 60 * 10
    prevTime = None
    
    # Contiguous messages.
    for line in conversations:
        contents = line.strip().split("[SEP]")
        sender = contents[1].replace(",", "")
        message = " ".join(contents[2:])
        timestamp = contents[0]
        if not currentSender:
            currentSender = sender
            prevTime = int(time.mktime(time.strptime(timestamp, pattern)))
        tokens = nltk.word_tokenize(message)
        currentTime = int(time.mktime(time.strptime(timestamp, pattern)))
        if sender == currentSender and currentTime - prevTime <= gapTime:
            currentTokens += tokens
            currentMessage += message + " "
            prevTime = currentTime
            continue
        else:
            feats = word_feats(currentTokens)
            probs = classifier.prob_classify(feats)
            pos = probs.prob("pos")
            row = {"sentiment": pos, "sender": sender}
            writer.writerow(row)
            sentimentScores.append((pos, currentMessage[:-1]))
            currentSender = sender
            prevTime = currentTime
            currentMessage = message + " "
            currentTokens = tokens
    
    # Last batch.
    feats = word_feats(currentTokens)
    probs = classifier.prob_classify(feats)
    pos = probs.prob("pos")
    row = {"sentiment": pos, "sender": sender}
    writer.writerow(row)
    
    sentimentScores.append((pos, currentMessage[:-1]))
    
    sentimentScores.sort(key = lambda message: message[0], reverse = True)
    
    output = open("Files/contiguousMessageSentimentScores", "w")
    for message in sentimentScores:
        print("{0}: {1}".format(message[0], message[1]), file = output)
    
    output.close()

def printConversation(i, conversationsDict):
    for message in conversationsDict[i]:
        print(message)

def conversationSentimentAnalysis(classifier):
    print("Getting conversation sentiment scores...")
    
    conversations = open("Conversations.txt")
    
    oldMessageTime = None
    conversationsDict = {0: []}
    conversation = 0
    conversationGap = 60 * 60
    pattern = "%Y-%m-%d %H:%M:%S"
    line = conversations.readline().strip()
    contents = line.split("[SEP]")
    timestamp = contents[0]
    firstTime = int(time.mktime(time.strptime(timestamp, pattern)))
    senders = {}
    
    for line in conversations:
        contents = line.strip().split("[SEP]")
        timestamp = contents[0]
        sender = contents[1].replace(",", "")
        if sender not in senders:
            senders[sender] = True
        curMessageTime = int(time.mktime(time.strptime(timestamp, pattern)))
        if not oldMessageTime:
            oldMessageTime = curMessageTime
        diff = curMessageTime - oldMessageTime
        if diff > conversationGap:
            conversation += 1<commit_msg>Still fixing things.<commit_after>from __future__ import print_function

import csv
import nltk
import sys
import time

from nltk.classify import NaiveBayesClassifier

def word_feats(words):
    return dict([(word, True) for word in words])

def runSentimentAnalysis(checkContiguousMessages = True, checkConversations = True, checkWeeks = True):
    
    print("Performing sentiment analysis...")
    
    f = open("sentimentTraining")
    tweets = [line.strip() for line in f]
    totalTweets = len(tweets)
    
    features = []
    
    count = 0
    
    # Takes a while.
    for tweet in tweets:
        if count % 10000 == 0:
            percent = 100 * count / totalTweets
            sys.stdout.write("\rTraining {0}% complete...".format(percent))
            sys.stdout.flush()
        [sentiment, sentence] = tweet.split("[SEP]")
        # sentence = unicode(sentence, 'utf-8')
        posOrNeg = None
        if sentiment == "0":
            posOrNeg = "neg"
        elif sentiment == "4":
            posOrNeg = "pos"
        else:
            continue
        tokens = nltk.word_tokenize(sentence)
        feats = word_feats(tokens)
        features.append((feats, posOrNeg))
        count += 1
    
    print()
    classifier = NaiveBayesClassifier.train(features)
    
    classifier.show_most_informative_features(100)
    
    messageSentimentAnalysis(classifier)
    if checkContiguousMessages:
        contiguousMessagesSentimentAnalysis(classifier)
    
    if checkConversations:
        conversationSentimentAnalysis(classifier)
    
    if checkWeeks:
        weeklySentimentAnalysis(classifier)

def messageSentimentAnalysis(classifier):
    print("Getting message sentiment scores...")
    
    conversations = open("Conversations.txt")
    
    fieldNames = ["sentiment", "sender"]
    writer = csv.DictWriter(open("Files/messageSentiments.csv", "w"), fieldNames)
    writer.writeheader()
    
    sentimentScores = []
    
    # Individual messages.
    for line in conversations:
        contents = line.strip().split("[SEP]")
        sender = contents[1].replace(",", "")
        message = " ".join(contents[2:])
        tokens = nltk.word_tokenize(message)
        feats = word_feats(tokens)
        probs = classifier.prob_classify(feats)
        pos = probs.prob("pos")
        row = {"sentiment": pos, "sender": sender}
        writer.writerow(row)
        sentimentScores.append((pos, message))
    
    sentimentScores.sort(key = lambda message: message[0], reverse = True)
    
    output = open("Files/messageSentimentScores", "w")
    for message in sentimentScores:
        print("{0}: {1}".format(message[0], message[1]), file = output)
    
    output.close()

def contiguousMessagesSentimentAnalysis(classifier):
    print("Getting contiguous messages sentiment scores...")
    
    conversations = open("Conversations.txt")
    
    fieldNames = ["sentiment", "sender"]
    writer = csv.DictWriter(open("Files/contiguousMessageSentiments.csv", "w"), fieldNames)
    writer.writeheader()
    
    sentimentScores = []
    currentSender = None
    currentMessage = ""
    currentTokens = []
    pattern = "%Y-%m-%d %H:%M:%S"
    gapTime = 60 * 10
    prevTime = None
    
    # Contiguous messages.
    for line in conversations:
        contents = line.strip().split("[SEP]")
        sender = contents[1].replace(",", "")
        message = " ".join(contents[2:])
        timestamp = contents[0]
        if not currentSender:
            currentSender = sender
            prevTime = int(time.mktime(time.strptime(timestamp, pattern)))
        tokens = nltk.word_tokenize(message)
        currentTime = int(time.mktime(time.strptime(timestamp, pattern)))
        if sender == currentSender and currentTime - prevTime <= gapTime:
            currentTokens += tokens
            currentMessage += message + " "
            prevTime = currentTime
            continue
        else:
            feats = word_feats(currentTokens)
            probs = classifier.prob_classify(feats)
            pos = probs.prob("pos")
            row = {"sentiment": pos, "sender": sender}
            writer.writerow(row)
            sentimentScores.append((pos, currentMessage[:-1]))
            currentSender = sender
            prevTime = currentTime
            currentMessage = message + " "
            currentTokens = tokens
    
    # Last batch.
    feats = word_feats(currentTokens)
    probs = classifier.prob_classify(feats)
    pos = probs.prob("pos")
    row = {"sentiment": pos, "sender": sender}
    writer.writerow(row)
    
    sentimentScores.append((pos, currentMessage[:-1]))
    
    sentimentScores.sort(key = lambda message: message[0], reverse = True)
    
    output = open("Files/contiguousMessageSentimentScores", "w")
    for message in sentimentScores:
        print("{0}: {1}".format(message[0], message[1]), file = output)
    
    output.close()

def printConversation(i, conversationsDict):
    for message in conversationsDict[i]:
        print(message)

def conversationSentimentAnalysis(classifier):
    print("Getting conversation sentiment scores...")
    
    conversations = open("Conversations.txt")
    
    oldMessageTime = None
    conversationsDict = {0: []}
    conversation = 0
    conversationGap = 60 * 60
    pattern = "%Y-%m-%d %H:%M:%S"
    senders = {}
    
    for line in conversations:
        contents = line.strip().split("[SEP]")
        timestamp = contents[0]
        sender = contents[1].replace(",", "")
        if sender not in senders:
            senders[sender] = True
        curMessageTime = int(time.mktime(time.strptime(timestamp, pattern)))
        if not oldMessageTime:
            oldMessageTime = curMessageTime
        diff = curMessageTime - oldMessageTime
        if diff > conversationGap:
            conversation += 1
{'fraction_non_alphanumeric': 0.11642675698560542, 'fraction_numerical': 0.008213378492802709, 'mean_word_length': 2.572595281306715, 'pattern_counts': {'":': 16, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5152551', 'n_tokens_mistral': 3449, 'n_tokens_neox': 3404, 'n_words': 738}