<commit_before>﻿from datetime import timedelta
import geometry
import Levenshtein
import MantelTest
import matplotlib.pyplot as plt
import numpy
import page
from random import shuffle
from scipy import log, log2, mean, polyfit, sqrt, stats, std
import scipy.cluster
import re

chain_codes = [["A", "B", "C", "D"], ["E", "F", "G", "H"]]

#############################################################################
# MEASURE LEARNABILITY: CALCULATE TRANSMISSION ERROR AND THEN COMPARE THE
# VERIDICAL SCORE TO A MONTE CARLO SAMPLE

def learnability(experiment, chain, generation, simulations=100000):
    words_A = getWords(experiment, chain, generation, "s")
    words_B = getWords(experiment, chain, generation-1, "s")
    x = meanNormLevenshtein(words_A, words_B)
    m, sd = MonteCarloError(words_A, words_B, simulations)
    z = (m-x)/sd
    return x, m, sd, z

# CALCULATE THE TRANSMISSION ERROR BETWEEN TWO CONSECUTIVE PARTICIPANTS

def transmissionError(experiment, chain, generation):
    words_A = getWords(experiment, chain, generation, "s")
    words_B = getWords(experiment, chain, generation-1, "s")
    return meanNormLevenshtein(words_A, words_B)

# GIVEN TWO SETS OF STRINGS, SHUFFLE ONE SET OF STRINGS n TIMES. COMPUTE THE
# MEAN NORMALIZED LEVENSHTEIN DISTANCE FOR EACH SHUFFLING AND RETURN THE MEAN
# AND STANDARD DEVIATION OF THE COMPUTED SCORES

def MonteCarloError(strings1, strings2, simulations):
    distances = []
    for i in xrange(0, simulations):
        shuffle(strings1)
        distances.append(meanNormLevenshtein(strings1, strings2))
    return mean(distances), std(distances)

# CALCULATE THE MEAN NORMALIZED LEVENSHTEIN DISTANCE BETWEEN TWO SETS OF STRINGS

def meanNormLevenshtein(strings1, strings2):
    total = 0.0
    for i in range(0, len(strings1)):
        ld = Levenshtein.distance(strings1[i], strings2[i])
        total += ld/float(max(len(strings1[i]), len(strings2[i])))
    return total/float(len(strings1))

#############################################################################
# MEASURE LEARNABILITY IN TRAINING: CALCULATE THE MEAN NORMALIZED LEVENSHTEIN
# DISTANCE FOR A SPECIFIC INDIVIDUAL'S TRAINING RESULTS

def trainingError(experiment, chain, generation):
    data = load(experiment, chain, generation, "log")
    words_A = [data[x][0] for x in range(5,53)]
    words_B = [data[x][1] for x in range(5,53)]
    x = meanNormLevenshtein(words_A, words_B)
    return x

#############################################################################
# COUNT THE NUMBER OF UNIQUE WORDS FOR GIVEN PARTICIPANT

def uniqueStrings(experiment, chain, generation):
    dynamic_data = load(experiment, chain, generation, "d")
    stable_data = load(experiment, chain, generation, "s")
    dynamic_words = [row[0] for row in dynamic_data]
    stable_words = [row[0] for row in stable_data]
    combined_words = dynamic_words + stable_words
    return len(set(dynamic_words)), len(set(stable_words)), len(set(combined_words))

#############################################################################
# GET TRANSMISSION ERROR RESULTS FOR ALL CHAINS IN AN EXPERIMENT

def allTransmissionErrors(experiment):
    results = []
    for chain in chain_codes[experiment-1]:
        scores = []
        for generation in range(1, 11):
            score = transmissionError(experiment, chain, generation)
            scores.append(score)
        results.append(scores)
    return results

#############################################################################
# GET LEARNABILITY RESULTS FOR ALL CHAINS IN AN EXPERIMENT

def allLearnability(experiment, sims=100000):
    results = []
    for chain in chain_codes[experiment-1]:
        print "Chain " + chain + "..."
        seed(RandomDotOrg().get_seed())
        scores = []
        for generation in range(1, 11):
            score = learnability(experiment, chain, generation, sims)[3]
            scores.append(score)
        results.append(scores)
    return results

#############################################################################
# GET TRANSMISSION ERROR RESULTS FOR ALL CHAINS IN AN EXPERIMENT

def allTrainingErrors(experiment):
    results = []
    for chain in chain_codes[experiment-1]:
        scores = []
        for generation in range(1, 11):
            score = trainingError(experiment, chain, generation)
            scores.append(score)
        results.append(scores)
    return results

#############################################################################
# PLOT MEANS FOR EACH GENERATION WITH ERROR BARS (95% CI)

def plotMean(matrix, start=1, y_label="Score", miny=0.0, maxy=1.0, conf=False):
    fig, ax = plt.subplots(figsize=plt.figaspect(0.625))
    m = len(matrix)
    n = len(matrix[0])
    if conf == True:
        ax.plot(range(-1,n+2), [1.959964] * (n+3), color='gray', linestyle=':')
        ax.plot(range(-1,n+2), [-1.959964] * (n+3), color='gray', linestyle=':')
        ax.plot(range(-1,n+2), [2.734369] * (n+3), color='k', linestyle='--')
        ax.plot(range(-1,n+2), [-2.734369] * (n+3), color='k', linestyle='--')
    means = []
    errors = []    
    for i in range(0,n):
        column = [row[i] for row in matrix if row[i] != None]
        means.append(mean(column))
        errors.append((std(column)/sqrt(m))*1.959964)
    xvals = range(start, n+start)
    (_, caps, _) = ax.errorbar(xvals, means, yerr=errors, color='k', linestyle="-", linewidth=2.0, capsize=5.0, elinewidth=1.5)
    for cap in caps:
        cap.set_markeredgewidth(2)
    labels = range(start, start+n)
    plt.xticks(xvals, labels, fontsize=14)
    plt.yticks(fontsize=14)
    plt.xlim(start-0.5, n+start-0.5)
    plt.ylim(miny, maxy)
    plt.xlabel("Generation number", fontsize=22)
    plt.ylabel(y_label, fontsize=22)
    plt.show()

#############################################################################
# PLOT ALL CHAINS FROM A DATA MATRIX

def plotAll(matrix, start=1, y_label="Score", miny=0.0, maxy=1.0, conf=False):
    fig, ax = plt.subplots(figsize=plt.figaspect(0.625))
    colours = ["#2E578C","#5D9648","#E7A13D","#BC2D30"]
    n = len(matrix[0])
    xvals = range(start, n+start)
    if conf == True:
        ax.plot(range(0,n+1), [1.959964] * (n+1), color='gray', linestyle=':')
        ax.plot(range(0,n+1), [-1.959964] * (n+1), color='gray', linestyle=':')
        ax.plot(range(0,n+1), [2.734369] * (n+1), color='k', linestyle='--')
        ax.plot(range(0,n+1), [-2.734369] * (n+1), color='k', linestyle='--')
    for i in range(0,len(matrix)):
        x_vals = range(start, len(matrix[i])+start)
        ax.plot(x_vals, matrix[i], color=colours[i], linewidth=2.0)
    labels = range(start, start+n)
    plt.xlim(start, n+start-1)
    plt.ylim(miny, maxy) 
    plt.xticks(xvals, labels, fontsize=14)
    plt.yticks(fontsize=14)
    plt.xlabel("Generation number", fontsize=22)
    plt.ylabel(y_label, fontsize=22)
    plt.show()

#############################################################################
# RUN ALL THREE STATS

def runStats(data_matrix, hypothesis="d", include_gen_zero=True):

    if include_gen_zero == False:
        matrix = []
        for row in data_matrix:
            matrix.append([row[i] for i in range(1,len(row))])
    else:
        matrix = data_matrix<commit_msg>Separate options for confidence levels<commit_after>﻿from datetime import timedelta
import geometry
import Levenshtein
import MantelTest
import matplotlib.pyplot as plt
import numpy
import page
from random import shuffle
from scipy import log, log2, mean, polyfit, sqrt, stats, std
import scipy.cluster
import re

chain_codes = [["A", "B", "C", "D"], ["E", "F", "G", "H"]]

#############################################################################
# MEASURE LEARNABILITY: CALCULATE TRANSMISSION ERROR AND THEN COMPARE THE
# VERIDICAL SCORE TO A MONTE CARLO SAMPLE

def learnability(experiment, chain, generation, simulations=100000):
    words_A = getWords(experiment, chain, generation, "s")
    words_B = getWords(experiment, chain, generation-1, "s")
    x = meanNormLevenshtein(words_A, words_B)
    m, sd = MonteCarloError(words_A, words_B, simulations)
    z = (m-x)/sd
    return x, m, sd, z

# CALCULATE THE TRANSMISSION ERROR BETWEEN TWO CONSECUTIVE PARTICIPANTS

def transmissionError(experiment, chain, generation):
    words_A = getWords(experiment, chain, generation, "s")
    words_B = getWords(experiment, chain, generation-1, "s")
    return meanNormLevenshtein(words_A, words_B)

# GIVEN TWO SETS OF STRINGS, SHUFFLE ONE SET OF STRINGS n TIMES. COMPUTE THE
# MEAN NORMALIZED LEVENSHTEIN DISTANCE FOR EACH SHUFFLING AND RETURN THE MEAN
# AND STANDARD DEVIATION OF THE COMPUTED SCORES

def MonteCarloError(strings1, strings2, simulations):
    distances = []
    for i in xrange(0, simulations):
        shuffle(strings1)
        distances.append(meanNormLevenshtein(strings1, strings2))
    return mean(distances), std(distances)

# CALCULATE THE MEAN NORMALIZED LEVENSHTEIN DISTANCE BETWEEN TWO SETS OF STRINGS

def meanNormLevenshtein(strings1, strings2):
    total = 0.0
    for i in range(0, len(strings1)):
        ld = Levenshtein.distance(strings1[i], strings2[i])
        total += ld/float(max(len(strings1[i]), len(strings2[i])))
    return total/float(len(strings1))

#############################################################################
# MEASURE LEARNABILITY IN TRAINING: CALCULATE THE MEAN NORMALIZED LEVENSHTEIN
# DISTANCE FOR A SPECIFIC INDIVIDUAL'S TRAINING RESULTS

def trainingError(experiment, chain, generation):
    data = load(experiment, chain, generation, "log")
    words_A = [data[x][0] for x in range(5,53)]
    words_B = [data[x][1] for x in range(5,53)]
    x = meanNormLevenshtein(words_A, words_B)
    return x

#############################################################################
# COUNT THE NUMBER OF UNIQUE WORDS FOR GIVEN PARTICIPANT

def uniqueStrings(experiment, chain, generation):
    dynamic_data = load(experiment, chain, generation, "d")
    stable_data = load(experiment, chain, generation, "s")
    dynamic_words = [row[0] for row in dynamic_data]
    stable_words = [row[0] for row in stable_data]
    combined_words = dynamic_words + stable_words
    return len(set(dynamic_words)), len(set(stable_words)), len(set(combined_words))

#############################################################################
# GET TRANSMISSION ERROR RESULTS FOR ALL CHAINS IN AN EXPERIMENT

def allTransmissionErrors(experiment):
    results = []
    for chain in chain_codes[experiment-1]:
        scores = []
        for generation in range(1, 11):
            score = transmissionError(experiment, chain, generation)
            scores.append(score)
        results.append(scores)
    return results

#############################################################################
# GET LEARNABILITY RESULTS FOR ALL CHAINS IN AN EXPERIMENT

def allLearnability(experiment, sims=100000):
    results = []
    for chain in chain_codes[experiment-1]:
        print "Chain " + chain + "..."
        seed(RandomDotOrg().get_seed())
        scores = []
        for generation in range(1, 11):
            score = learnability(experiment, chain, generation, sims)[3]
            scores.append(score)
        results.append(scores)
    return results

#############################################################################
# GET TRANSMISSION ERROR RESULTS FOR ALL CHAINS IN AN EXPERIMENT

def allTrainingErrors(experiment):
    results = []
    for chain in chain_codes[experiment-1]:
        scores = []
        for generation in range(1, 11):
            score = trainingError(experiment, chain, generation)
            scores.append(score)
        results.append(scores)
    return results

#############################################################################
# PLOT MEANS FOR EACH GENERATION WITH ERROR BARS (95% CI)

def plotMean(matrix, start=1, y_label="Score", miny=0.0, maxy=1.0, conf=False, conf_bonf=False):
    fig, ax = plt.subplots(figsize=plt.figaspect(0.625))
    m = len(matrix)
    n = len(matrix[0])
    if conf == True:
        ax.plot(range(-1,n+2), [1.959964] * (n+3), color='gray', linestyle=':')
        ax.plot(range(-1,n+2), [-1.959964] * (n+3), color='gray', linestyle=':')
    if conf_bonf == True:
        ax.plot(range(-1,n+2), [2.734369] * (n+3), color='k', linestyle='--')
        ax.plot(range(-1,n+2), [-2.734369] * (n+3), color='k', linestyle='--')
    means = []
    errors = []    
    for i in range(0,n):
        column = [row[i] for row in matrix if row[i] != None]
        means.append(mean(column))
        errors.append((std(column)/sqrt(m))*1.959964)
    xvals = range(start, n+start)
    (_, caps, _) = ax.errorbar(xvals, means, yerr=errors, color='k', linestyle="-", linewidth=2.0, capsize=5.0, elinewidth=1.5)
    for cap in caps:
        cap.set_markeredgewidth(2)
    labels = range(start, start+n)
    plt.xticks(xvals, labels, fontsize=14)
    plt.yticks(fontsize=14)
    plt.xlim(start-0.5, n+start-0.5)
    plt.ylim(miny, maxy)
    plt.xlabel("Generation number", fontsize=22)
    plt.ylabel(y_label, fontsize=22)
    plt.show()

#############################################################################
# PLOT ALL CHAINS FROM A DATA MATRIX

def plotAll(matrix, start=1, y_label="Score", miny=0.0, maxy=1.0, conf=False, conf_bonf=False):
    fig, ax = plt.subplots(figsize=plt.figaspect(0.625))
    colours = ["#2E578C","#5D9648","#E7A13D","#BC2D30"]
    n = len(matrix[0])
    xvals = range(start, n+start)
    if conf == True:
        ax.plot(range(0,n+1), [1.959964] * (n+1), color='gray', linestyle=':')
        ax.plot(range(0,n+1), [-1.959964] * (n+1), color='gray', linestyle=':')
    if conf_bonf == True
        ax.plot(range(0,n+1), [2.734369] * (n+1), color='k', linestyle='--')
        ax.plot(range(0,n+1), [-2.734369] * (n+1), color='k', linestyle='--')
    for i in range(0,len(matrix)):
        x_vals = range(start, len(matrix[i])+start)
        ax.plot(x_vals, matrix[i], color=colours[i], linewidth=2.0)
    labels = range(start, start+n)
    plt.xlim(start, n+start-1)
    plt.ylim(miny, maxy) 
    plt.xticks(xvals, labels, fontsize=14)
    plt.yticks(fontsize=14)
    plt.xlabel("Generation number", fontsize=22)
    plt.ylabel(y_label, fontsize=22)
    plt.show()

#############################################################################
# RUN ALL THREE STATS

def runStats(data_matrix, hypothesis="d", include_gen_zero=True):

    if include_gen_zero == False:
        matrix = []
        for row in data_matrix:
            matrix.append([row[i] for i in range(1,len(row))])
    else:
        matrix = data_matrix
{'fraction_non_alphanumeric': 0.21869452410214393, 'fraction_numerical': 0.028949883927352177, 'mean_word_length': 4.123119972018189, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '44551', 'n_tokens_mistral': 5110, 'n_tokens_neox': 4806, 'n_words': 1155}