<jupyter_start><jupyter_text>Extract turn pairs for all dialogues<jupyter_code># load from pickles
adam_sent_turn_pairs = pickle.load(open('data/pickles/adam_sent_turn-pairs.pkl', 'rb'))
adam_pos_turn_pairs = pickle.load(open('data/pickles/adam_pos_turn-pairs.pkl', 'rb'))
sarah_sent_turn_pairs = pickle.load(open('data/pickles/sarah_sent_turn-pairs.pkl', 'rb'))
sarah_pos_turn_pairs = pickle.load(open('data/pickles/sarah_pos_turn-pairs.pkl', 'rb'))

sarah_ages = list(sorted(sarah_sent_turn_pairs.keys()))
adam_ages = list(sorted(adam_sent_turn_pairs.keys()))<jupyter_output><empty_output><jupyter_text>Exclude turnsExclude any turn pairs that have a turn with mean utterance length < 2.  Compute measures and matrices<jupyter_code># load from pickles
adam_measures = pickle.load(open('data/pickles/adam-measures.pkl', 'rb'))
sarah_measures = pickle.load(open('data/pickles/sarah-measures.pkl', 'rb'))<jupyter_output><empty_output><jupyter_text>Encoding sentences with BERT<jupyter_code>from sentence_transformers import SentenceTransformer
model = SentenceTransformer('bert-base-nli-mean-tokens')
from scipy import spatial
def cosine_sim(vec1, vec2):
    distance = spatial.distance.cosine
    return 1 - distance(vec1,vec2)

A = list(model.encode(['I love the field of computational dialog modelling'])[0])
B = list(model.encode(['I really enjoy computationally modelling dialogue'])[0])

cosine_sim(A,B)
import copy
sarah_sent_turn_vectors

def turn2Vec(corpus):
    sent_turn_vectors = copy.deepcopy(corpus)
    no_dialogues = len(corpus.keys())

    for i, age_key in enumerate(sent_turn_vectors.keys()):
        dialogue = sent_turn_vectors[age_key]
        for pair in dialogue:
            for turn in pair:
                sent = turn[-1][0]
                sent = [' '.join(sent)]
                sent = list(model.encode(sent)[0])
                turn[-1][0] = sent
        print("{} out of {} dialogues encoded".format(i+1, no_dialogues))
    return sent_turn_vectors

sarah_vectors = turn2Vec(sarah_sent_turn_pairs)
adam_vectors = turn2Vec(adam_sent_turn_pairs)
pickle.dump(sarah_vectors, open('sarah_vectors.pkl', 'wb'))
pickle.dump(adam_vectors, open('adam_vectors.pkl', 'wb'))
adam_vectors = pickle.load(open('adam_vectors.pkl', 'rb'))
sarah_vectors = pickle.load(open('sarah_vectors.pkl', 'rb'))
def cosim_matrix(turn_pairs):
    """

    """
    
    N_pairs = len(turn_pairs)
    
    turns_j = [t[1] for t in turn_pairs]
    j_idx = np.repeat(np.arange(N_pairs), N_pairs)
    turns_i = [t[0] for t in turn_pairs]
    i_idx = np.tile(np.arange(N_pairs), N_pairs)
    
    m_vec = np.array([cosine_sim(turns_i[i][1][0], turns_j[j][1][0])
                      for j, i in zip(j_idx, i_idx)])
    
    m_mat = m_vec.reshape(N_pairs,N_pairs)
    
    return m_mat
example_matrix<jupyter_output><empty_output><jupyter_text>Testing:<jupyter_code>A = model.encode([' '.join(sarah_sent_turn_pairs[51.43][0][0][1][0])])[0]
B = model.encode([' '.join(sarah_sent_turn_pairs[51.43][0][1][1][0])])[0]
cosine_sim(A,B)<jupyter_output><empty_output><jupyter_text>Plotting<jupyter_code>len(sarah_vectors.keys())
def plot_rec_matrix(rec_mat, cmap, title, filename=False):
    """
    Plot recurrence matrix (like Fernandez & Grimm 2014, Fig.1)
    on alignment measures or on difference in complexity measures.
    """
    plt.figure()
    plt.imshow(rec_mat, cmap=cmap)
    plt.colorbar()
    plt.gca().invert_yaxis()
    plt.xlabel('adult turns')
    plt.ylabel('child turns')
    plt.title(title)
    if filename:
        plt.savefig("figures/recurrence_plots/adam_semantic_281", dpi=300)
        plt.close()
    else:
        plt.show()

# age = 28.56
# subset = 99999
# for age in sorted(sarah_vectors.keys()):
#     pos_al = cosim_matrix(sarah_vectors[age])#[:subset, :subset]

#     plot_rec_matrix(pos_al, cmap='Greys', 
#                     title="Sarah: Cosine Similarity (age " + str(age) + ")")
pos_al = cosim_matrix(adam_vectors[28.1])

plot_rec_matrix(pos_al, cmap='Greys', 
                title="Adam: Cosine Similarity (age " + str(age) + ")", filename=True)
def local_rate(matrix, d, subset=None):
    """
    This should be RR_d
    """
    # we go over the diagonal of the matrix
    # i.e. consider all points (i, j) where i == j
    # and then we also take the points that are
    # d steps away (above or below) from the diagonal,
    # i.e. from (j, i-d) to (j, i+d)
    # over all those points, we take the mean
    
    # i is i-th turn by participant on the x-axis (adult)
    # j is j-th turn by participant on the y-axis (child)
    
    n = len(matrix)
    
    D = []
    
    # iterate over the diagonal
    for i, j in zip(range(n), range(n)):
        
        # only consider points where j > i
        if subset == 'pos':
            if i < d:
                D.extend(matrix[j, :i])
            else:
                D.extend(matrix[j, i-d:i])
            
        # only consider points where i > j
        elif subset == 'neg':
            if i < d:
                D.extend(matrix[j, i+1:i+1+d])
            else:
                D.extend(matrix[j, i+1:i+1+d])
            
        # consider points in both directions
        else:
            if i < d:
                D.extend(matrix[j, :i+d+1])
            else:
                D.extend(matrix[j, i-d:i+d+1])
                
    return np.nanmean(D)

# age = 28.56
# matrix = cosim_matrix(sarah_vectors[age])


def RR_over_time(turn_vectors, shuffle):
    RR_dict = {}
    for age in sorted(turn_vectors.keys()):
        if shuffle:
            matrix = cosim_matrix(turn_vectors[age])
            np.random.shuffle(matrix)
        else:
            matrix = cosim_matrix(turn_vectors[age])
        
        RR_dict[age] = {}
        for d in range(0, 11, 1):
            print('finished computing dialogue at age: {} RR for d: {}'.format(age, d))
            RR_dict[age][d] = local_rate(matrix=matrix, d=d)

    return RR_dict

# sarah_RRs = []
# for d in range(0, 11, 1):
#     sarah_RR = RR_over_time(sarah_vectors,d)
#     sarah_RRs.append(sarah_RR)
#     print('finished computing RR for d: {}'.format(d))

sarah_RRs_shf = RR_over_time(sarah_vectors, shuffle=True)
adam_RRs = RR_over_time(adam_vectors, shuffle=False)
adam_RRs_shf = RR_over_time(adam_vectors, shuffle=True)
pickle.dump(sarah_RRs, open('sarah_RRs.pkl', 'wb'))
pickle.dump(sarah_RRs_shf, open('sarah_RRs_shf.pkl', 'wb'))
pickle.dump(adam_RRs, open('adam_RRs.pkl', 'wb'))
pickle.dump(adam_RRs_shf, open('adam_RRs_shf.pkl', 'wb'))
def dict_2_list(dictionary):
    RR_per_d = [[],[],[],[],[],[],[],[],[],[],[]]

    for age in sorted(dictionary.keys()):
        for d in sorted(dictionary[age].keys()):
            RR_per_d[d].append(dictionary[age][d])
    return RR_per_d

Sarah_per_d = dict_2_list(sarah_RRs)
Sarah_per_d_shf = dict_2_list(sarah_RRs_shf)
Adam_per_d = dict_2_list(adam_RRs)
Adam_per_d_shf = dict_2_list(adam_RRs_shf)
def plot_locRR(locRR, locRR_shf, title, filename=False):
    """
    Plot the local RR (averaged over all dialogues) at different
    levels of d in original vs. shuffled dialogues.
    (see Fernandez & Grimm 2014, Fig. 2)
    """
    plt.figure()
    xs = [int(d) for d in range(0,11)]
    y = [np.mean(locRR[d]) for d in range(0,11)]
    yerr = [nansem(locRR[d]) for d in range(0,11)]
    
    y_shf = [np.mean(locRR_shf[d]) for d in range(0,11)]
    yerr_shf = [nansem(locRR_shf[d]) for d in range(0,11)]
    plt.errorbar(xs, y, yerr, color='crimson', label='original')
    plt.errorbar(xs, y_shf, yerr_shf, color='teal', label='shuffled')
    plt.xlabel('$d$')
    plt.ylabel('$RR_d$')
    plt.title(title)
    plt.legend()
    if filename:
        plt.savefig("figures/local_RR/" + filename, dpi=300)
        plt.close()
    else:
        plt.show()

locRR = Sarah_per_d
locRR_shf = Sarah_per_d_shf
    
plot_locRR(Sarah_per_d, Sarah_per_d_shf, title="Sarah: Semantic Convergence")#, filename="sarah_convergence")
plot_locRR(Adam_per_d, Adam_per_d_shf, title="Adam: Semantic Convergence")#, filename="adam_convergence")
ages = []
convergence = []
d=0
for age in sorted(sarah_RRs.keys()):
    ages.append(age)
    convergence.append(sarah_RRs[age][d]-sarah_RRs_shf[age][d])

x = ages
y = convergence
plt.figure()
plt.title('semantic recurrence over time for d = 0')
plt.xlabel('age')
plt.ylabel('mean RR')
plt.plot(x,y)
adam_RRs<jupyter_output><empty_output><jupyter_text>Convergence in complexity  Local RR for varying d(something resembling Fig. 2 from Fernandez & Grimm 2014)<jupyter_code>def plot_locRR(locRR, locRR_shf, title, filename=False):
    """
    Plot the local RR (averaged over all dialogues) at different
    levels of d in original vs. shuffled dialogues.
    (see Fernandez & Grimm 2014, Fig. 2)
    """
    plt.figure()
    xs = [int(d) for d in sorted(locRR.keys())]
    y = [np.mean(locRR[d]) for d in sorted(locRR.keys())]
    yerr = [nansem(locRR[d]) for d in sorted(locRR.keys())]
    
    y_shf = [np.mean(locRR_shf[d]) for d in sorted(locRR_shf.keys())]
    yerr_shf = [nansem(locRR_shf[d]) for d in sorted(locRR_shf.keys())]
    plt.errorbar(xs, y, yerr, color='crimson', label='original')
    plt.errorbar(xs, y_shf, yerr_shf, color='teal', label='shuffled')
    plt.xlabel('$d$')
    plt.ylabel('$RR_d$')
    plt.title(title)
    plt.legend()
    if filename:
        plt.savefig("figures/local_RR/" + filename, dpi=300)
        plt.close()
    else:
        plt.show()

locRR = sarah_RRs
locRR_shf = sarah_measures['PosBi']['shLocRR']
    
plot_locRR(locRR, locRR_shf, title="Sarah: POS bigrams")<jupyter_output><empty_output><jupyter_text>Paired t-tests for positive & negative RR_2Fernandez & Grimm 2014, Table 3Is recurrence significantly higher when the adult's turn follows the child's?<jupyter_code># Mean word length: no
mwl_RR2pos = sarah_measures['MWL']['RR2pos']
mwl_RR2neg = sarah_measures['MWL']['RR2neg']
ttest_rel(mwl_RR2pos, mwl_RR2neg)
# Mean utterance length: no
mul_RR2pos = sarah_measures['MUL']['RR2pos']
mul_RR2neg = sarah_measures['MUL']['RR2neg']
ttest_rel(mul_RR2pos, mul_RR2neg)
# Lexeme unigrams: yes!
# RR2pos is significantly smaller than RR2neg
# because t < 0; this is a 2-tailed test, for 1-tailed divide p by 2
lexuni_RR2pos = sarah_measures['LexUni']['RR2pos']
lexuni_RR2neg = sarah_measures['LexUni']['RR2neg']
ttest_rel(lexuni_RR2pos, lexuni_RR2neg)
# Lexeme bigrams: yes!
# RR2pos is significantly smaller than RR2neg
# because t < 0; this is a 2-tailed test, for 1-tailed divide p by 2
lexbi_RR2pos = sarah_measures['LexBi']['RR2pos']
lexbi_RR2neg = sarah_measures['LexBi']['RR2neg']
ttest_rel(lexbi_RR2pos, lexbi_RR2neg)
# POS bigrams: yes!
# RR2pos is significantly smaller than RR2neg
# because t < 0; this is a 2-tailed test, for 1-tailed divide p by 2
posbi_RR2pos = sarah_measures['PosBi']['RR2pos']
posbi_RR2neg = sarah_measures['PosBi']['RR2neg']
ttest_rel(posbi_RR2pos, posbi_RR2neg)<jupyter_output><empty_output><jupyter_text>Plot all the plots!!!Save to figures directory<jupyter_code># child_name = 'Adam'
# child_ages = adam_ages
# measures = adam_measures

# # complexity convergence
# plot_chi_adt(measures['MWL']['CHI'], measures['MWL']['ADT'], 
#              'MWL', title=child_name + ": Child vs. Adult word lengths",
#              filename=child_name + "_CHI-ADT_MWL.png")
# plot_chi_adt(measures['MUL']['CHI'], measures['MUL']['ADT'], 
#              'MUL', title=child_name + ": Child vs. Adult utterance lengths",
#              filename=child_name + "_CHI-ADT_MUL.png")
# plot_chi_adt_time(child_ages, measures['MWL']['CHI'], measures['MWL']['ADT'], 
#                   'MWL', title=child_name + ": Mean word length over time",
#                   filename=child_name + "_CHI-ADT_time_MWL.png")
# plot_chi_adt_time(child_ages, measures['MUL']['CHI'], measures['MUL']['ADT'], 
#                   'MUL', title=child_name + ": Mean utterance length over time",
#                   filename=child_name + "_CHI-ADT_time_MUL.png")

# # global recurrence over time
# plot_globRR(child_ages, measures['MWL']['GlobRR'], 'MWL',
#             title=child_name + ": Global MWL recurrence over time",
#             filename=child_name + "_globRR_MWL.png")
# plot_globRR(child_ages, measures['MUL']['GlobRR'], 'MUL',
#             title=child_name + ": Global MUL recurrence over time",
#             filename=child_name + "_globRR_MUL.png")
# plot_globRR(child_ages, measures['LexUni']['GlobRR'], 'Lexeme Unigrams',
#             title=child_name + ": Global Lexeme Unigram recurrence over time",
#             filename=child_name + "_globRR_LexUni.png")
# plot_globRR(child_ages, measures['LexBi']['GlobRR'], 'Lexeme Bigrams',
#             title=child_name + ": Global Lexeme Bigram recurrence over time",
#             filename=child_name + "_globRR_LexBi.png")
# plot_globRR(child_ages, measures['PosBi']['GlobRR'], 'POS Bigrams',
#             title=child_name + ": Global POS Bigram recurrence over time",
#             filename=child_name + "_globRR_PosBi.png")

# # recurrence plots
# for age in child_ages:
#     plot_rec_matrix(measures['MWL']['matrix'][age], cmap='RdYlGn',
#                     title=child_name + ": Mean word length (age " + 
#                     str(round(age,2)) + "mo)",
#                     filename=child_name + "/MWL_" + str(round(age,2)) + ".png")
#     plot_rec_matrix(measures['MUL']['matrix'][age], cmap='RdYlGn',
#                     title=child_name + ": Mean utterance length (age " + 
#                     str(round(age,2)) + "mo)",
#                     filename=child_name + "/MUL_" + str(round(age,2)) + ".png")
    
#     plot_rec_matrix(measures['LexUni']['matrix'][age], cmap='Greys',
#                     title=child_name + ": Lexeme Unigrams (age " + 
#                     str(round(age,2)) + "mo)",
#                     filename=child_name + "/LexUni_" + str(round(age,2)) + ".png")
#     plot_rec_matrix(measures['LexBi']['matrix'][age], cmap='Greys',
#                     title=child_name + ": Lexeme Bigrams (age " + 
#                     str(round(age,2)) + "mo)",
#                     filename=child_name + "/LexBi_" + str(round(age,2)) + ".png")
#     plot_rec_matrix(measures['PosBi']['matrix'][age], cmap='Greys',
#                     title=child_name + ": POS Bigrams (age " + 
#                     str(round(age,2)) + "mo)",
#                     filename=child_name + "/PosBi_" + str(round(age,2)) + ".png")
    
# # local RR for varying d
# plot_locRR(measures['MWL']['LocRR'], measures['MWL']['shLocRR'], 
#            title=child_name + ": Mean word length",
#            filename=child_name + "_locRR_MWL.png")
# plot_locRR(measures['MUL']['LocRR'], measures['MUL']['shLocRR'], 
#            title=child_name + ": Mean utterance length",
#            filename=child_name + "_locRR_MUL.png")
# plot_locRR(measures['LexUni']['LocRR'], measures['LexUni']['shLocRR'], 
#            title=child_name + ": Lexical Unigrams",
#            filename=child_name + "_locRR_LexUni.png")
# plot_locRR(measures['LexBi']['LocRR'], measures['LexBi']['shLocRR'], 
#            title=child_name + ": Lexical Bigrams",
#            filename=child_name + "_locRR_LexBi.png")
# plot_locRR(measures['PosBi']['LocRR'], measures['PosBi']['shLocRR'], 
#            title=child_name + ": POS Bigrams",
#            filename=child_name + "_locRR_PosBi.png")
# name = 'sarah'
# dlg_files = sarah_dlgs
# sent_turn_pairs = sarah_sent_turn_pairs
# postag_turn_pairs = sarah_pos_turn_pairs
# excl_idx = sarah_excl_idx

# d_range = range(11)

# measures = {'ages': [],
#             'MWL': {'CHI': [], 'ADT': [], 
#                     'matrix': {}, 'shuffled': {},
#                     'GlobRR': [], 'RR2pos': [], 'RR2neg': [],
#                     'LocRR': {d: [] for d in d_range},
#                     'shLocRR': {d: [] for d in d_range}},
#             'MUL': {'CHI': [], 'ADT': [], 
#                     'matrix': {}, 'shuffled': {},
#                     'GlobRR': [], 'RR2pos': [], 'RR2neg': [],
#                     'LocRR': {d: [] for d in d_range},
#                     'shLocRR': {d: [] for d in d_range}},
#             'LexUni': {'matrix': {}, 'shuffled': {},
#                        'GlobRR': [], 'RR2pos': [], 'RR2neg': [],
#                        'LocRR': {d: [] for d in d_range},
#                        'shLocRR': {d: [] for d in d_range}},
#             'LexBi': {'matrix': {}, 'shuffled': {},
#                       'GlobRR': [], 'RR2pos': [], 'RR2neg': [],
#                       'LocRR': {d: [] for d in d_range},
#                       'shLocRR': {d: [] for d in d_range}},
#             'PosBi': {'matrix': {}, 'shuffled': {},
#                       'GlobRR': [], 'RR2pos': [], 'RR2neg': [],
#                       'LocRR': {d: [] for d in d_range},
#                       'shLocRR': {d: [] for d in d_range}}}

# for dlg_file in dlg_files:
    
#     # get dialogue sentences and POS tags
#     dlg_sents = brown.sents(dlg_file)
#     tagged_sents = brown.tagged_sents(dlg_file)
#     dlg_postags = [[spkID, [w[1] for w in utt]]
#               for spkID, utt in tagged_sents]
    
#     # child's age
#     age = round(float(brown.age(dlg_file, month=True)[0]), 2)
#     if age in measures['ages']:
#         age += 0.01
#     measures['ages'].append(age)
#     print(age)
    
#     # compute child's and adults' mean word lengths
#     dlg_mwordlen = dlg_mwl(dlg_sents)
#     chi_mwordlen = [utt[1] for utt in dlg_mwordlen 
#                     if utt[0] == 'CHI']
#     adt_mwordlen = [utt[1] for utt in dlg_mwordlen 
#                     if utt[0] != 'CHI']
#     measures['MWL']['CHI'].append((np.nanmean(chi_mwordlen), nansem(chi_mwordlen)))
#     measures['MWL']['ADT'].append((np.nanmean(adt_mwordlen), nansem(adt_mwordlen)))
    
#     # compute child's and adults' mean utterance lengths
#     dlg_muttlen = dlg_mul(dlg_sents)
#     chi_muttlen = [utt[1] for utt in dlg_muttlen 
#                     if utt[0] == 'CHI']
#     adt_muttlen = [utt[1] for utt in dlg_muttlen 
#                     if utt[0] != 'CHI']
#     measures['MUL']['CHI'].append((np.nanmean(chi_muttlen), nansem(chi_muttlen)))
#     measures['MUL']['ADT'].append((np.nanmean(adt_muttlen), nansem(adt_muttlen)))    
    
#     # matrices and shuffled matrices for MWL and MUL
#     mwl_tp = [t for i, t in enumerate(turn_pairs(dlg_mwl(dlg_sents)))
#               if i not in excl_idx]
#     mwl_rec_mat = diff_matrix(mwl_tp)
#     shf_mwl_mat = np.copy(mwl_rec_mat.flat)
#     np.random.shuffle(shf_mwl_mat)
#     shf_mwl_mat = shf_mwl_mat.reshape(mwl_rec_mat.shape)
#     measures['MWL']['matrix'][age] = mwl_rec_mat
#     measures['MWL']['shuffled'][age] = shf_mwl_mat
    
#     mul_tp = [t for i, t in enumerate(turn_pairs(dlg_mul(dlg_sents)))
#              if i not in excl_idx]
#     mul_rec_mat = diff_matrix(mul_tp)
#     shf_mul_mat = np.copy(mul_rec_mat.flat)
#     np.random.shuffle(shf_mul_mat)
#     shf_mul_mat = shf_mul_mat.reshape(mul_rec_mat.shape)
#     measures['MUL']['matrix'][age] = mul_rec_mat
#     measures['MUL']['shuffled'][age] = shf_mwl_mat
    
#     # matrices and shuffled matrices for lexical unigrams and bigrams
#     lexuni_rec_mat = conv_matrix(sent_turn_pairs[age], n=1)
#     shf_lexuni_mat = np.copy(lexuni_rec_mat.flat)
#     np.random.shuffle(shf_lexuni_mat)
#     shf_lexuni_mat = shf_lexuni_mat.reshape(lexuni_rec_mat.shape)
#     measures['LexUni']['matrix'][age] = lexuni_rec_mat
#     measures['LexUni']['shuffled'][age] = shf_lexuni_mat
    
#     lexbi_rec_mat = conv_matrix(sent_turn_pairs[age], n=2)
#     shf_lexbi_mat = np.copy(lexbi_rec_mat.flat)
#     np.random.shuffle(shf_lexbi_mat)
#     shf_lexbi_mat = shf_lexbi_mat.reshape(lexbi_rec_mat.shape)
#     measures['LexBi']['matrix'][age] = lexbi_rec_mat
#     measures['LexBi']['shuffled'][age] = shf_lexbi_mat
    
#     # matrix and shuffled matrix for pos bigrams
#     posbi_rec_mat = conv_matrix(postag_turn_pairs[age], n=2)
#     shf_posbi_mat = np.copy(posbi_rec_mat.flat)
#     np.random.shuffle(shf_posbi_mat)
#     shf_posbi_mat = shf_posbi_mat.reshape(posbi_rec_mat.shape)
#     measures['PosBi']['matrix'][age] = posbi_rec_mat
#     measures['PosBi']['shuffled'][age] = shf_posbi_mat
    
#     # global recurrence rates
#     measures['MWL']['GlobRR'].append(global_rate(mwl_rec_mat))
#     measures['MUL']['GlobRR'].append(global_rate(mul_rec_mat))
#     measures['LexUni']['GlobRR'].append(global_rate(lexuni_rec_mat))
#     measures['LexBi']['GlobRR'].append(global_rate(lexbi_rec_mat))
#     measures['PosBi']['GlobRR'].append(global_rate(posbi_rec_mat))
    
#     # positive and negative local recurrence rates (d = 2)
#     measures['MWL']['RR2pos'].append(local_rate(mwl_rec_mat, d=2, subset='pos'))
#     measures['MWL']['RR2neg'].append(local_rate(mwl_rec_mat, d=2, subset='neg'))
#     measures['MUL']['RR2pos'].append(local_rate(mul_rec_mat, d=2, subset='pos'))
#     measures['MUL']['RR2neg'].append(local_rate(mul_rec_mat, d=2, subset='neg'))
#     measures['LexUni']['RR2pos'].append(local_rate(lexuni_rec_mat, d=2, subset='pos'))
#     measures['LexUni']['RR2neg'].append(local_rate(lexuni_rec_mat, d=2, subset='neg'))
#     measures['LexBi']['RR2pos'].append(local_rate(lexbi_rec_mat, d=2, subset='pos'))
#     measures['LexBi']['RR2neg'].append(local_rate(lexbi_rec_mat, d=2, subset='neg'))
#     measures['PosBi']['RR2pos'].append(local_rate(posbi_rec_mat, d=2, subset='pos'))
#     measures['PosBi']['RR2neg'].append(local_rate(posbi_rec_mat, d=2, subset='neg'))
    
#     # local recurrence rates for d between 0 and 10,
#     # for shuffled and unshuffled matrices
#     for d in d_range:
#         measures['MWL']['LocRR'][d].append(local_rate(mwl_rec_mat, d=d))
#         measures['MWL']['shLocRR'][d].append(local_rate(shf_mwl_mat, d=d))
#         measures['MUL']['LocRR'][d].append(local_rate(mul_rec_mat, d=d))
#         measures['MUL']['shLocRR'][d].append(local_rate(shf_mul_mat, d=d))
#         measures['LexUni']['LocRR'][d].append(local_rate(lexuni_rec_mat, d=d))
#         measures['LexUni']['shLocRR'][d].append(local_rate(shf_lexuni_mat, d=d))
#         measures['LexBi']['LocRR'][d].append(local_rate(lexbi_rec_mat, d=d))
#         measures['LexBi']['shLocRR'][d].append(local_rate(shf_lexbi_mat, d=d))
#         measures['PosBi']['LocRR'][d].append(local_rate(posbi_rec_mat, d=d))
#         measures['PosBi']['shLocRR'][d].append(local_rate(shf_posbi_mat, d=d))
    
#     pickle.dump(measures, open(name + '-measures.pkl', 'wb'))
# # load files for Adam or Sarah
# name = 'sarah'
# dlg_files = sarah_dlgs

# ages = []
# sent_turn_pairs = {}
# postag_turn_pairs = {}

# # extract turn pairs (for words and POS tags) and pickle to file
# for dlg_file in dlg_files:
#     dlg_sents = brown.sents(dlg_file)
#     tagged_sents = brown.tagged_sents(dlg_file)
#     dlg_postags = [[spkID, [w[1] for w in utt]]
#               for spkID, utt in tagged_sents]
    
#     age = round(float(brown.age(dlg_file, month=True)[0]), 2)
#     if age in ages:
#         age += 0.01
#     ages.append(age)
#     print(age)
    
#     sent_turnp = turn_pairs(turns(dlg_sents))
#     sent_turn_pairs[age] = sent_turnp
    
#     postag_turnp = turn_pairs(turns(dlg_postags))
#     postag_turn_pairs[age] = postag_turnp
    
#     pickle.dump(sent_turn_pairs, open(name + '_sent_turn-pairs.pkl', 'wb'))
#     pickle.dump(postag_turn_pairs, open(name + '_pos_turn-pairs.pkl', 'wb'))
# def idx_to_exclude(turn_pairs):
#     """
#     Returns indices of turn pairs to exclude
#     (those that contain a turn for which the MUL is
#     lower than 2)
#     """
#     exclude = []
#     for idx, (turn_i, turn_j) in enumerate(turn_pairs):
#         mul_i = np.mean([len(u) for u in turn_i[1]])
#         mul_j = np.mean([len(u) for u in turn_j[1]])
#         if mul_i < 2 or mul_j < 2:
#             exclude.append(idx)
#     return exclude

# excl_pct = []
# for age in adam_sent_turn_pairs.keys():
#     adam_excl_idx = idx_to_exclude(adam_sent_turn_pairs[age])
    
#     pct = round((len(adam_excl_idx)/len(adam_sent_turn_pairs[age]))*100,1)
#     excl_pct.append(pct)
#     # print percentage of each dialogue excluded
#     # print(pct, '%')
#     adam_sent_turn_pairs[age] = [t for i, t in enumerate(adam_sent_turn_pairs[age])
#                                 if i not in adam_excl_idx]
#     adam_pos_turn_pairs[age] = [t for i, t in enumerate(adam_pos_turn_pairs[age])
#                                if i not in adam_excl_idx]
# print(np.mean(excl_pct), '% of Adam corpus excluded')

# excl_pct = []
# for age in sarah_sent_turn_pairs.keys():
#     sarah_excl_idx = idx_to_exclude(sarah_sent_turn_pairs[age])
    
#     pct = round((len(sarah_excl_idx)/len(sarah_sent_turn_pairs[age]))*100,1)
#     excl_pct.append(pct)
#     # print percentage of each dialogue excluded
#     # print(pct, '%')
#     sarah_sent_turn_pairs[age] = [t for i, t in enumerate(sarah_sent_turn_pairs[age])
#                                 if i not in sarah_excl_idx]
#     sarah_pos_turn_pairs[age] = [t for i, t in enumerate(sarah_pos_turn_pairs[age])
#                                if i not in sarah_excl_idx]
# print(np.mean(excl_pct), '% of Sarah corpus excluded')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1558070279928529, 'fraction_numerical': 0.010363311494937463, 'mean_word_length': 3.3148877848209697, 'pattern_counts': {'":': 19, '<': 42, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '640309', 'license': 'MIT', 'n_tokens_mistral': 9714, 'n_tokens_neox': 9280, 'n_words': 1703, 'path': 'semantic_recurrence.ipynb', 'repo_name': 'mdhk/CDM-project', 'size': '25185'}