<jupyter_start><jupyter_text>Functions<jupyter_code>def getModuAll(adjDict,taus,pRandRewires,repetitions = 100):
    
    lenTaus = len(taus)
    lenPRand = len(pRandRewires)
    Q = np.zeros((lenPRand,lenTaus,repetitions))  
    
    for rep in np.arange(repetitions):
        for indT,tau in enumerate(taus):
            for indP,p in enumerate(pRandRewires):
        
                #load the specific rewired matrix
                A = adjDict[rep+1][(p, tau, 4000)][1]
        
                #construct it so that igraph can read it
                #make it undirected
                g = ig.Graph(directed=False)
                #make it weighted
                g.es["weight"] = 1.0                
                
                g.add_vertices(len(A))
                ix, jx = A.nonzero()
                for i,j in zip(ix, jx):
                    if i<j:
                        g.add_edge(i, j, weight=A[i,j])
            
                #calculate the clusters and their modularity score    
                clusters = g.community_multilevel(weights=g.es['weight'])
                modularity_score = g.modularity(clusters.membership,weights=g.es['weight'])
        
                #store it in the matrix
                Q[indP,indT,rep] = modularity_score
                
    return Q<jupyter_output><empty_output><jupyter_text>Manipulate data to the the Q matrices and to save them -- LOGNORMAL<jupyter_code>#parameters tested
rewirings = 4000
pRand = [0,0.2]

taus = [5.4,5.45,5.5,5.55,5.6,5.65,5.7,5.75,5.8,5.85,5.9,5.95,6]
weightDist = 'lognormal'

###### Load Adjacency matrices
directoryALoad ='data/ArandA/500iterations/'

#filePathALoad = directoryALoad + 'ArandA_tauTransition500_'+weightDist+'_'+str(rewirings)+'.pckl'
filePathALoad = directoryALoad + 'ArandA_tauTransition250firstHalf_'+weightDist+'_'+str(rewirings)+'.pckl'
filePathALoad2 = directoryALoad + 'ArandA_tauTransition250secondHalf_'+weightDist+'_'+str(rewirings)+'.pckl'

ArandA = hf.loadVar(filePathALoad)
ArandA2 = hf.loadVar(filePathALoad2)

Q1 = getModuAll(ArandA,taus,pRand,250)
Q2 = getModuAll(ArandA2,taus,pRand,250)

Q = np.concatenate((Q1, Q2), axis=2)
Q2.shape

directoryQSave ='data/ModularityValues/500iterations/'
descr = 'QTransition_'
filePathQSave = directoryQSave + descr+weightDist+'.pckl'
hf.saveVarSimple((Q,taus), filePathQSave)<jupyter_output><empty_output><jupyter_text>Normal -get the Q<jupyter_code>rewirings = 4000
pRand = [0,0.2]
taus = np.array([4.0,4.05,4.1,4.15,4.2,4.25,4.3])

directoryLoad ='data/ArandA/500iterations/'
weightDist = 'normal'

filePathLoad = directoryLoad + 'ArandA_tauTransition_'+weightDist+'_'+str(rewirings)+'.pckl'

ArandA = hf.loadVar(filePathLoad)
#Calculate the modularity values
Q = getModuAll(ArandA,taus,pRand,500)
directoryQSave ='data/ModularityValues/500iterations/'
descr = 'QTransition_'
filePathQSave = directoryQSave + descr+weightDist+'.pckl'
hf.saveVarSimple((Q,taus), filePathQSave)
Q.shape
Q[:,:,100]<jupyter_output><empty_output><jupyter_text>Add the 0 variation to tau var AArand<jupyter_code>weightDist = 'normal'
p =  0.2

tauSpread = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.,1.1,1.2,1.3,1.4,1.5]
#tauCenter = [5.8]*len(tauSpread)
#tauCenter = [5.5]*len(tauSpread)
#tauCenter = [4.2]*len(tauSpread)
tauCenter = [4.15]*len(tauSpread)

#tauDistribution = ['normal']*len(tauSpread)
tauDistribution = ['uniform']*len(tauSpread)

tauTuples = list(zip(tauCenter,tauSpread,tauDistribution))

directoryVarLoad ='data/ArandA/variableTau/'
filePathVarLoad = directoryVarLoad + 'ArandA_tauTransition_'+weightDist+'_pRand'+str(p)+tauDistribution[0]+'.pckl'

tauTarget = tauCenter[0]
directoryLoad = 'data/ArandA/'
rewirings = 4000
filePathLoad = directoryLoad + 'ArandA_tauTransition_'+weightDist+'_'+str(rewirings)+'.pckl'

ArandAVar = hf.loadVar(filePathVarLoad)

ArandA = hf.loadVar(filePathLoad)
tausIter = [tauCenter[0]]*4000
for it in ArandA.keys():
    kk = (ArandA[it][p,tauCenter[0],4000][0],ArandA[it][p,tauCenter[0],4000][1],tausIter)
    ArandAVar[it][p,(tauCenter[0],0,tauDistribution[0]),4000] = kk

ArandAVar[2].keys()
ArandA[1].keys()
hf.saveVarSimple(ArandAVar, filePathVarLoad)<jupyter_output><empty_output><jupyter_text>Normal<jupyter_code>#parameters tested
rewirings = 4000
pRand = [0,0.2]

taus1 = np.array([4.1,4.15,4.2,4.25])
taus2 = np.array([4.0,4.05,4.3])

weightDist = 'normal'

###### Load Adjacency matrices
directoryALoad ='data/ArandA/500iterations/'

#filePathALoad = directoryALoad + 'ArandA_tauTransition500_'+weightDist+'_'+str(rewirings)+'.pckl'
filePathALoad = directoryALoad + 'ArandA_tauTransition500_'+weightDist+'_'+str(rewirings)+'.pckl'
filePathALoad2 = directoryALoad + 'ArandA_2tauTransition500_'+weightDist+'_'+str(rewirings)+'.pckl'

ArandA = hf.loadVar(filePathALoad)
ArandA2 = hf.loadVar(filePathALoad2)
ArandA_all = {}
for k in ArandA:
    ArandA_all[k] = {}
    ArandA_all[k].update(ArandA[k])
    ArandA_all[k].update(ArandA2[k])
directorySave ='data/ArandA/500iterations/'

weightDist = 'normal'

filePathSave = directorySave + 'ArandA_tauTransition_'+weightDist+'_'+str(rewirings)+'.pckl'
hf.saveVarSimple(ArandA_all, filePathSave)<jupyter_output><empty_output><jupyter_text>Miscellanious<jupyter_code>#taus = np.arange(3.5,4.55,0.05)
#taus = np.array([4.1,4.15,4.2,4.25])
#taus = np.arange(5.35,6.01,0.05)
taus = np.array([4.0,4.05,4.3])
#taus = np.arange(4.5,6.55,0.05)
pRand = [0,0.2]
#parameters tested
rewirings = 4000
pRand = [0,0.2]
taus = [5.4,5.45,5.5,5.55,5.6,5.65,5.7,5.75,5.8,5.85,5.9,5.95,6]


#weightDist = 'normal'
weightDist = 'lognormal'

###### Load Adjacency matrices
directoryALoad ='data/ArandA/500iterations/'


#filePathALoad = directoryALoad + 'ArandA_tauTransition500_'+weightDist+'_'+str(rewirings)+'.pckl'
filePathALoad = directoryALoad + 'ArandA_tauTransition500firstHalf_'+weightDist+'_'+str(rewirings)+'.pckl'

ArandA = hf.loadVar(filePathALoad)

#######Path to save modularity values
#directoryQSave ='data/ModularityValues/'
#descr = 'QAll_'
#descr = 'QTransition_'
#filePathQSave = directoryQSave + descr+weightDist+'.pckl'
kk_actual = list(ArandA[1].keys())

kk_correct = [(0, 5.35, 4000),(0.2, 5.35, 4000),(0, 5.4, 4000),(0.2, 5.4, 4000),(0, 5.45, 4000),(0.2, 5.45, 4000),(0, 5.5, 4000),
 (0.2, 5.5, 4000),(0, 5.55, 4000),(0.2, 5.55, 4000),(0, 5.6, 4000),(0.2, 5.6, 4000),(0, 5.65, 4000),(0.2, 5.65, 4000),
 (0, 5.7, 4000),(0.2, 5.7, 4000),(0, 5.75, 4000),(0.2, 5.75, 4000),(0, 5.8, 4000),(0.2, 5.8, 4000),(0, 5.85, 4000),
 (0.2, 5.85, 4000),(0, 5.9, 4000),(0.2, 5.9, 4000),(0, 5.95, 4000),(0.2, 5.95, 4000),(0, 6, 4000),(0.2, 6, 4000)]
for it in ArandA.keys():
    for ind,k in enumerate(list(ArandA[it].keys())):
        if (np.abs(k[1] - kk_correct[ind][1])<0.01)&(np.abs(k[0] - kk_correct[ind][0])<0.0001): 
            ArandA[it][kk_correct[ind]] = ArandA[it][k]
            del ArandA[it][k]
        else:
            print('there is something wrong')

hf.saveVarSimple(ArandA, filePathALoad)
for k in ArandA[1].keys():
    print(k)<jupyter_output>(0, 5.4, 4000)
(0.2, 5.4, 4000)
(0, 5.45, 4000)
(0.2, 5.45, 4000)
(0, 5.5, 4000)
(0.2, 5.5, 4000)
(0, 5.55, 4000)
(0.2, 5.55, 4000)
(0, 5.6, 4000)
(0.2, 5.6, 4000)
(0, 5.65, 4000)
(0.2, 5.65, 4000)
(0, 5.7, 4000)
(0.2, 5.7, 4000)
(0, 5.75, 4000)
(0.2, 5.75, 4000)
(0, 5.8, 4000)
(0.2, 5.8, 4000)
(0, 5.85, 4000)
(0.2, 5.85, 4000)
(0, 5.9, 4000)
(0.2, 5.9, 4000)
(0, 5.95, 4000)
(0.2, 5.95, 4000)
(0, 6, 4000)
(0.2, 6, 4000)<jupyter_text>Miscellanious 2<jupyter_code>list(ArandA[1].keys())[4:]
kk_correct = [(0, 3.6, 4000),(0.2, 3.6, 4000),(0, 3.65, 4000),(0.2, 3.65, 4000),(0, 3.7, 4000),(0.2, 3.7, 4000),(0, 3.75, 4000),(0.2, 3.75, 4000),
(0, 3.8, 4000),(0.2, 3.8, 4000),(0, 3.85, 4000),(0.2, 3.85, 4000),(0, 3.9, 4000),(0.2, 3.9, 4000),(0, 3.95, 4000),
(0.2, 3.95, 4000),(0, 4, 4000),(0.2, 4, 4000),(0, 4.05, 4000),(0.2, 4.05, 4000),(0, 4.1, 4000),(0.2, 4.1, 4000),
(0, 4.15, 4000),(0.2, 4.15, 4000),(0, 4.2, 4000),(0.2, 4.2, 4000),(0, 4.25, 4000),(0.2, 4.25, 4000),(0, 4.3, 4000),
(0.2, 4.3, 4000),(0, 4.35, 4000),(0.2, 4.35, 4000),(0, 4.4, 4000),(0.2, 4.4, 4000),(0, 4.45, 4000),(0.2, 4.45, 4000),
(0, 4.5, 4000),(0.2, 4.5, 4000)]
kk_correct = [(0, 4.65, 4000),(0.2, 4.65, 4000),(0, 4.7, 4000),(0.2, 4.7, 4000),(0, 4.75, 4000),(0.2, 4.75, 4000),
(0, 4.8, 4000),(0.2, 4.8, 4000),(0, 4.85, 4000),(0.2, 4.85, 4000),(0, 4.9, 4000),(0.2, 4.9, 4000),(0, 4.95, 4000),
(0.2, 4.95, 4000),(0, 5, 4000),(0.2, 5, 4000),(0, 5.05, 4000),(0.2, 5.05, 4000),(0, 5.1, 4000),(0.2, 5.1, 4000),
(0, 5.15, 4000),(0.2, 5.15, 4000),(0, 5.2, 4000),(0.2, 5.2, 4000),(0, 5.25, 4000),(0.2, 5.25, 4000),(0, 5.3, 4000),
(0.2, 5.3, 4000),(0, 5.35, 4000),(0.2, 5.35, 4000),(0, 5.4, 4000),(0.2, 5.4, 4000),(0, 5.45, 4000),(0.2, 5.45, 4000),
(0, 5.5, 4000),(0.2, 5.5, 4000),(0, 5.55, 4000),(0.2, 5.55, 4000),(0, 5.6, 4000),(0.2, 5.6, 4000),(0, 5.65, 4000),
(0.2, 5.65, 4000),(0, 5.7, 4000),(0.2, 5.7, 4000),(0, 5.75, 4000),(0.2, 5.75, 4000),(0, 5.8, 4000),(0.2, 5.8, 4000),
(0, 5.85, 4000), (0.2, 5.85, 4000),(0, 5.9, 4000),(0.2, 5.9, 4000),(0, 5.95, 4000),(0.2, 5.95, 4000),
(0, 6, 4000),(0.2, 6, 4000),(0, 6.05, 4000),(0.2, 6.05, 4000),(0, 6.1, 4000),(0.2, 6.1, 4000),(0, 6.15, 4000),
(0.2, 6.15, 4000),(0, 6.2, 4000),(0.2, 6.2, 4000),(0, 6.25, 4000),(0.2, 6.25, 4000),(0, 6.3, 4000),(0.2, 6.3, 4000),
(0, 6.35, 4000),(0.2, 6.35, 4000),(0, 6.4, 4000),(0.2, 6.4, 4000),(0, 6.45, 4000),(0.2, 6.45, 4000),(0, 6.5, 4000),
(0.2, 6.5, 4000)]
for it in ArandA.keys():
    for ind,k in enumerate(list(ArandA[it].keys())[4:]):
        if (np.abs(k[1] - kk_correct[ind][1])<0.01)&(np.abs(k[0] - kk_correct[ind][0])<0.0001): 
            ArandA[it][kk_correct[ind]] = ArandA[it][k]
            del ArandA[it][k]
        else:
            print(k)
for k in ArandA[1].keys():
    print(k)
hf.saveVarSimple(ArandA, filePathLoad)<jupyter_output><empty_output><jupyter_text>Fix the tau approximations<jupyter_code>rewirings = 4000
pRand = [0,0.2]
  
wD = 'lognormal'
###### Load Adjacency matrices
directoryALoad ='data/ArandA/'

filePathLoad = directoryALoad + 'ArandA_'+wD+'_'+str(rewirings)+'.pckl'
ArandA = hf.loadVar(filePathLoad)    
ArandA[1].keys()
kk_correct = [(0, 0, 4000), (0.2, 0, 4000), (0, 0.2, 4000), (0.2, 0.2, 4000), (0, 0.4, 4000), (0.2, 0.4, 4000), 
           (0, 0.6, 4000), (0.2, 0.6, 4000), (0, 0.8, 4000), (0.2, 0.8, 4000), (0, 1, 4000), (0.2, 1, 4000), 
           (0, 1.2, 4000), (0.2, 1.2, 4000), (0, 1.4, 4000), (0.2, 1.4, 4000), (0, 1.6, 4000), (0.2, 1.6, 4000), 
           (0, 1.8, 4000), (0.2, 1.8, 4000), (0, 2, 4000), (0.2, 2, 4000), (0, 2.2, 4000), (0.2, 2.2, 4000), 
           (0, 2.4, 4000), (0.2, 2.4, 4000), (0, 2.6, 4000), (0.2, 2.6, 4000), (0, 2.8, 4000), (0.2, 2.8, 4000), 
           (0, 3, 4000), (0.2, 3, 4000), (0, 3.2, 4000), (0.2, 3.2, 4000), (0, 3.4, 4000), (0.2, 3.4, 4000), 
           (0, 3.6, 4000), (0.2, 3.6, 4000), (0, 3.8, 4000), (0.2, 3.8, 4000), (0, 4, 4000), (0.2, 4, 4000), 
           (0, 4.2, 4000), (0.2, 4.2, 4000), (0, 4.4, 4000), (0.2, 4.4, 4000), (0, 4.6, 4000), (0.2, 4.6, 4000), 
           (0, 4.8, 4000), (0.2, 4.8, 4000), (0, 5, 4000), (0.2, 5, 4000), (0, 5.2, 4000), (0.2, 5.2, 4000), 
           (0, 5.4, 4000), (0.2, 5.4, 4000), (0, 5.6, 4000), (0.2, 5.6, 4000), (0, 5.8, 4000), (0.2, 5.8, 4000),
           (0, 6, 4000), (0.2, 6, 4000), (0, 6.2, 4000), (0.2, 6.2, 4000), (0, 6.4, 4000), (0.2, 6.4, 4000), 
           (0, 6.6, 4000), (0.2, 6.6, 4000), (0, 6.8, 4000), (0.2, 6.8, 4000), (0, 7, 4000), (0.2, 7, 4000), 
           (0, 7.2, 4000), (0.2, 7.2, 4000), (0, 7.4, 4000), (0.2, 7.4, 4000), (0, 7.6, 4000), (0.2, 7.6, 4000), 
           (0, 7.8, 4000), (0.2, 7.8, 4000), (0, 8, 4000), (0.2, 8, 4000)]
ArandANew = {}
for it in ArandA.keys():
    ArandANew[it] = {}
    for ind,k in enumerate(list(ArandA[it].keys())):
        if (np.abs(k[1] - kk_correct[ind][1])<0.01)&(np.abs(k[0] - kk_correct[ind][0])<0.0001): 
            ArandANew[it][kk_correct[ind]] = ArandA[it][k]
        else:
            print(k)
ArandANew[1].keys()
hf.saveVarSimple(ArandANew, filePathLoad)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2353185480521638, 'fraction_numerical': 0.19171027493977905, 'mean_word_length': 3.703125, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '571921', 'license': 'MIT', 'n_tokens_mistral': 7196, 'n_tokens_neox': 5532, 'n_words': 1056, 'path': '2GetSaveWFromBrokenUpDataSupplementary.ipynb', 'repo_name': 'rentzi/netRewireSpecFlexRob', 'size': '12039'}