<commit_before>import csv
import os
import glob
import time

from utils import file_utils, db_utils
from watson_developer_cloud import ToneAnalyzerV3

tone_analyzer = ToneAnalyzerV3(
    username='d6739ba1-970e-4b5a-9e38-f037fde2b299',
    password='<PASSWORD>',
    version='2016-02-11')


def main():
    committers = file_utils.load_emails('./groovy-coreteam.csv')
    cnx = db_utils.connect()
    base_dir = './emails'
    print ('building email corpus')
    db_utils.build_corpus(cnx, committers, base_dir)
    db_utils.disconnect(cnx)

    results_dir = './results'
    if not os.path.exists(results_dir):
        os.makedirs(results_dir)
    print ('analyzing tone')
    for dev in committers:
        file_name = '{0}/{1}.csv'.format(results_dir, dev['id'])
        with open(file_name, 'wb') as results:
            wr = csv.writer(results, delimiter=';')
            header = ('year-month', 'agreeableness')
            # social_tones = {'Openness', 'Conscientiousness', 'Extraversion', 'Agreeableness', 'Emotional Range'}
            wr.writerow(header)

            lookup = '{0}/{1}/*.txt'.format(base_dir, dev['id'])
            for email in glob.glob(lookup):
                y_m, _ = email.split('.txt', 1)
                _, y_m = y_m.split(base_dir + os.sep + dev['id'] + os.sep)
                with open(email, 'rb') as f:
                    content = f.read()
                    js = tone_analyze(content)
                    agreeableness_score = js['document_tone']['tone_categories'][0]['tones'][3]['score']
                    row = (y_m, agreeableness_score)
                    wr.writerow(row)
                    # Wait for a bit
                    time.sleep(.300)

        results.close()


def tone_analyze(text, tones='social', sentences=False):
    js = tone_analyzer.tone(text, tones, sentences)
    return js


if __name__ == "__main__":
    main()<commit_msg>final version<commit_after>import csv
import os
import glob
import time
import collections

from utils import file_utils, db_utils
from watson_developer_cloud import ToneAnalyzerV3

tone_analyzer = ToneAnalyzerV3(
    username='d6739ba1-970e-4b5a-9e38-f037fde2b299',
    password='<PASSWORD>',
    version='2016-02-11')


def main():
    committers = file_utils.load_emails('./groovy-coreteam.csv')
    cnx = db_utils.connect()
    base_dir = './emails'
    print ('building email corpus')
    #db_utils.build_corpus(cnx, committers, base_dir)
    db_utils.disconnect(cnx)

    results_dir = './results'
    if not os.path.exists(results_dir):
        os.makedirs(results_dir)
    print ('analyzing tone')
    for dev in committers:
        file_name = '{0}/{1}.csv'.format(results_dir, dev['id'])
        with open(file_name, 'wb') as results:
            wr = csv.writer(results, delimiter=';')
            header = ('year-month', 'agreeableness')
            # social_tones = {'Openness', 'Conscientiousness', 'Extraversion', 'Agreeableness', 'Emotional Range'}
            wr.writerow(header)

            rows = dict()
            lookup = '{0}/{1}/*.txt'.format(base_dir, dev['id'])
            for email in glob.glob(lookup):
                y_m, _ = email.split('.txt', 1)
                _, y_m = y_m.split(base_dir + os.sep + dev['id'] + os.sep)
                y, m = y_m.split('-')
                if len(m) == 1:
                    y_m = '{0}-0{1}'.format(y, m)
                with open(email, 'rb') as f:
                    content = f.read()
                    js = tone_analyze(content)
                    agreeableness_score = js['document_tone']['tone_categories'][0]['tones'][3]['score']
                    rows[y_m] = agreeableness_score
                    # Wait for a bit
                    time.sleep(.300)

            rows = collections.OrderedDict(sorted(rows.items()))
            for key, value in rows.items():
                wr.writerow([key, value])

        results.close()


def tone_analyze(text, tones='social', sentences=False):
    js = tone_analyzer.tone(text, tones, sentences)
    return js


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.13707699852869054, 'fraction_numerical': 0.020598332515939184, 'mean_word_length': 2.308191403081914, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5127927', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1278, 'n_words': 266}