<jupyter_start><jupyter_text><jupyter_code>!git clone https://github.com/radhikasethi2011/btechproj.git
import random
random.seed(10)
from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Mounted at /content/drive<jupyter_text>https://jaspar.genereg.net/matrix/MA0003.4/https://jaspar.genereg.net/sites/MA0003.4/ pipeline to prepare files:<jupyter_code>import random
import string
import numpy as np
from pathlib import Path
import pandas as pd

def get_num(a, b, x):
    """
    gets a unique number between a range (a,b), divisible by x
    input:  a - start inde
            b - end index
            x = random number should be divisible by x
    return:
        random number divisible by x
    """
    if not a % x:
        return random.choice(range(a, b, x))
    else:
        return random.choice(range(a + x - (a % x), b, x))

def motif_string():
    """
    generates a string of length (by get_num) of ACGT
    returns: string of equal ACGT
    """
    length = 288
    # print("random length: " , length)
    len4 = length / 4
    # print("len / 4 :" , len4)
    letters = ["A", "C", "G", "T"]
    letters_list = list(random.choice(letters))
    random.shuffle(letters_list)
    # print(" Random generated string with repetition:")
    str1 = ""
    return str1.join(letters_list)

def non_motif_string(length):
    """
    generates a string of length (by get_num) of ACGT
    returns: string 
    """
    # print("random length: " , length)
    #len4 = length / 4
    # print("len / 4 :" , len4)
    letters = ["A", "C", "G", "T"]
    nonmotiflist = []
    for i in range(length): 
      nonmotiflist.append(random.choice(letters))
    # print(" Random generated string with repetition:")
    str1 = ""
    return str1.join(nonmotiflist)

def add_motif():
    """
    adds a motif - string input, to output by non_motif_string
    returns: string consisting of a motif
    """
    motif = 'ATACGTTACCCG'
    result = list(motif_string())
    i = random.choice(range(len(result)))
    result.insert(i, motif)
    result = "".join(result)
    return result,i


def write_to_df(length,var): #to generate non motif background files
  l=[]
  for i1 in range(int(length)):
    l1=[]
    i=-5
    result= non_motif_string(var)
    seq = "seq_" + str(i1 + 1) + "_peak"
    a='0'      
    l1 = ["A",seq,str(i), result,a]
    l.append(list(l1))
  df = pd.DataFrame(l,columns=['FoldID',	'EventID',	'start_index',	'seq',	'Bound']) 
  return df

#write_to_file(1000)

#def write_to_df(length, var,name):

df_nonmotif_train = write_to_df(1000,314)
df_mixed_motif = write_to_df(1000,300)
df_mixed_nonmotif = write_to_df(1000,314)
df_motif_train = write_to_df(1000,300)
df_nonmotif_train.head()<jupyter_output><empty_output><jupyter_text>need to print lines 1 to 4 (0 is heading) of JASPAR file<jupyter_code>import numpy as np
from pathlib import Path


def read_pfm_jaspar(filepath_):
  with open(Path(filepath_)) as f:
    lines = f.readlines()

  a = []
  for i in range(1,5,1):
    b = list(lines[i].split()[2:-1])
    a.append(b)
  a = np.array(a,dtype=float)
  return a

pfm = read_pfm_jaspar('/content/MA0003.4.jaspar')
pfm
%cd /content/drive/MyDrive/BTech Proj/Data/MA0003.4
print(pfm.shape)
print(type(pfm))
import copy
ppm = copy.deepcopy(pfm)
print(ppm)
def convert_pfm_to_ppm(pfm):
  for i in range(pfm.shape[1]):
    sum = 0
    for j in range(pfm.shape[0]):
      sum+= pfm[j][i] 
    for j in range(pfm.shape[0]):
      prob = float(pfm[j][i]/sum)
      ppm[j][i] = float(prob)
  return(ppm)
  
p1=['A','C','G','T']
str1=''
def ppm_to_motif(ppm):
  motif = []
  mot=[]
  for i in range(ppm.shape[1]):
    arr = []
    for j in range(ppm.shape[0]):
      arr.append(float(ppm[j][i]))
    choice = np.random.choice(p1,p=arr) #motif is generated here
    mot.append(choice)
  
  return (''.join(mot))
  #return choice
ppm = convert_pfm_to_ppm(pfm)
print(ppm)

motif= ppm_to_motif(ppm)
print(motif)
for i in range(50):
  motif=ppm_to_motif(ppm)
  print(motif)<jupyter_output>ATGGCCTCACGCGA
GGAGCCTCAGGTGA
CCGGCCTCAGGCAT
ACACGCTCAGGGGG
ATTGCCTTAGGACG
ACTGCCACAGGACA
GTGGCCCCAGGCGA
TCTGCCTCAGGCCA
GCTTCCTCAGGGAT
CAGTCCTCAGGCCA
GTGGCCTCAGGGAT
GACGCCTCAGGCTA
TGCTCCTCAGGGCA
ATGGCCTGAGGCAA
CCTACCTCAGGGCT
GGAGCCTCAGGGAG
TTTGCCTCAGGGCC
AATGCCTCAGGCCA
GTTGCCTCCGGTGG
GGCCCCTCAGGGAG
CGGCCCTCAGGGTC
GGGCCCTCAGGCCT
TTTCCCACAGGCTT
ACTTCCTTAGGGTA
GCTGCCCCAGGGTG
GGATCCGAACGTCC
GTTCCCTCTGGCTC
ATGACCTCGGGAAC
CTTCCTTCAGGCTT
GTTGCCTCAGGGAC
CGCGCCTCTGGGGA
TTCCCCTCAGGCGA
GACGCCTCAGGCCG
CGAGCCTCAAGTAC
AGTGCCTCAGGCCC
GTCGCCGCAGGCAG
TGTGCCTCAGACTC
ACAGCCTCGGGCCG
GCCGCCTCAGGGCA
ACACCCTCAGGCTA
TTCGCCTCGGGCAG
TAACCCTCAGGATG
ACTGCCTCAGGCAC
CCAGCTTGAGGGGG
GCGGCTCCAGGCCC
TGTCCCTCCGGGAT
TCTCCCTCAGGCTA
ACTTCCTCAGGCAA
ACAGCCTCAGGCCA
CCACCCTCAGGCCT<jupyter_text>implanting motifs in 2 files - df_mixed_motif , df_motif_train<jupyter_code>#print(list(df_motif_train['seq']))
def non_motif_create(df1,df2): #to implant a  motif in a non motif file -> 1. motif file, 2. mixed file 
  mot=[]
  l1=[]
  l2=[]
  for i in range(0,len(df1)):
    motif = ppm_to_motif(ppm)
    mot.append(motif)
    index=random.randint(0,len(df1[i])-1) #0 to 300 
    l1.append(index)
    df1[i]=df1[i][:index] + str(motif) + df1[i][index:]
    
    index2=np.random.choice(len(df2[i]))
    l2.append(index2)
    df2[i]=df2[i][:index2] + str(motif) + df2[i][index2:]
    #print(index)
    #print(df1[i])
    #print(mot)
  return l1,l2,df1,df2,mot #l1, l2 to store the index
l1,l2,df_motif_train['seq'],df_mixed_motif['seq'],mot=non_motif_create(list(df_motif_train['seq']),list(df_mixed_motif['seq']))
len(l1)
len(df_motif_train['seq'][0])
df_motif_train['start_index'] = l1
df_mixed_motif['start_index'] = l2
df_motif_train['Bound']=df_motif_train['Bound'].astype('int32').replace(0,1)
df_mixed_motif['Bound']=df_mixed_motif['Bound'].astype('int32').replace(0,1)
df_mixed_motif['Bound'].dtype
df_mixed_motif.head()
df_motif_train.head()<jupyter_output><empty_output><jupyter_text>Creating mixed file using df_mixed_motif and df_mixed_nonmotif<jupyter_code>df_mixed_motif=df_mixed_motif.append(df_mixed_nonmotif,ignore_index=True)
df_mixed=df_mixed_motif.sample(frac=1).reset_index(drop=True)
df_mixed<jupyter_output><empty_output><jupyter_text>Saving all 3 files : df_mixed, df_motif_train, df_nonmotif_train<jupyter_code>%pwd
df_mixed.to_csv('/content/drive/MyDrive/BTech Proj/Data/MA0003.4/mixed.txt',index=None, sep='\t')
df_motif_train.to_csv('/content/drive/MyDrive/BTech Proj/Data/MA0003.4/motif.txt',index=None, sep='\t')
df_nonmotif_train.to_csv('/content/drive/MyDrive/BTech Proj/Data/MA0003.4/nonmotif.txt',index=None, sep='\t')<jupyter_output><empty_output><jupyter_text>now running through markov models<jupyter_code>import math
a = math.log(1)
a
pd.reset_option('all')
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve, auc
from sklearn.metrics import accuracy_score
import pandas as pd 
dict2, dict3 = {}, {}

def markov_model_motif(df1):
    """
    for a second degree motif markov model 
    returns dict2, dict3: count of the appearance of eg: A->C and eg: AC->T occuring
    """
    
    for j in range(len(df1)):
      seq = df1[j]
      for i in range(len(seq)-var): #dict2
        if seq[i:i+var] not in dict2: 
          dict2[seq[i:i+var]] = 1
        else: 
          dict2[seq[i:i+var]] += 1

      for i in range(len(seq)): #dict3
        if seq[i:i+var+1] not in dict3: 
          dict3[seq[i:i+var+1]] = 1
        else: 
          dict3[seq[i:i+var+1]] += 1

    #print('dict2: ', dict2)
    #print('dict3: ', dict3)
      
    return dict2, dict3


def motif_score1(dict2, dict3, seq):
    """
    returns the score of th markov model motif 
    """
    s = seq
    import math

    a=0
    sum = 0
    score = 1
    for i in range(0, len(s)):
      if s[i : i + var+1] not in dict3:
        #print("HELLOOOOOOOOOOOO", s[i : i + var+1])
        dict3[s[i : i + var+1]] = 1
      score = score * dict3[s[i : i + var+1]]
    #print('dict2-',dict2)
    #print('dict3-',dict3)
    return math.log(score),dict2,dict3

def motif_score(dict2, dict3, seq):
    """
    returns the score of th markov model motif 
    """
    s = seq
    import math
    score = 0
    ep=0.01
    for i in range(0, len(s)):
      if s[i : i + var+1] not in dict3:
        #print("HELLOOOOOOOOOOOO", s[i : i + var+1])
        dict3[s[i : i + var+1]] = ep
      else:
        dict3[s[i : i + var+1]] = dict3[s[i : i + var+1]] + ep
      score = score + math.log(dict3[s[i : i + var+1]])

    #print('dict2-',dict2)
    #print('dict3-',dict3)
    return score,dict2,dict3

def train(df,df2,df4):
  """
  df = for the motif markov model - seq 
  df2 = for the non motif markov model  - seq 

  df4 = the test file - pass the entire thing = seq + bound 

  Returns: the final dictionaries
  """
    # df1 = df['seq']
   
  dict2, dict3 = markov_model_motif(df)  # markov model for the motif.txt file
  # print("####################################################")
  
  dict2_non, dict3_non = markov_model_motif(df2)  # markov model for the non motif file
  # now, testing each sequence and predicting their class using the 2 dictionaries

  for i in range(len(df4)):
      list1 = []
      list2 = []
  for i in range(len(df4)):
      seq = df4["seq"][i]
      """
      1. score it using ddict3
      2. score it using dict3_non 
      3. whichever score higher, give value of that class 
      4. add seq, class to a list, append that list to the df 
      5. write this file 
      """

      score_motif,dict5,dict6 = motif_score(dict2, dict3, seq)
      score_nonmotif,dict7,dict8 = motif_score(dict2_non, dict3_non, seq)
      #print('for sequence: ', seq, 'score motif: ', score_motif, 'score_nonmotif: ', score_nonmotif)
      list1.append(score_motif-score_nonmotif)
      if score_motif > score_nonmotif:
          list2.append(1)
      else:
          list2.append(0)
  df4["predicted"] = list2
  df4["log odd score"] = list1
  display(df4)
  if score_motif > score_nonmotif:
    return dict6
  else:
    return dict8

  #print('dict2 - motif: ', dict5)
  #print('dict3 - motif : ', dict6)
 
    

def accuracy(df4):
    y_true = df4["Bound"]
    y_pred = df4["predicted"]
    return accuracy_score(y_true, y_pred)

colnames2 = ["FoldID", "EventID", "start_index", "seq", "Bound"]
colnames = ["FoldID", "EventID", "seq", "Bound"]
def true_false_positive(threshold_vector, y_test):
    true_positive = np.equal(threshold_vector, 1) & np.equal(y_test, 1)
    true_negative = np.equal(threshold_vector, 0) & np.equal(y_test, 0)
    false_positive = np.equal(threshold_vector, 1) & np.equal(y_test, 0)
    false_negative = np.equal(threshold_vector, 0) & np.equal(y_test, 1)

    tpr = true_positive.sum() / (true_positive.sum() + false_negative.sum())
    fpr = false_positive.sum() / (false_positive.sum() + true_negative.sum())

    return tpr, fpr


def roc_from_scratch(probabilities, y_test, partitions=100):
    roc = np.array([])
    for i in range(partitions + 1):
        
        threshold_vector = np.greater_equal(probabilities, i / partitions).astype(int)
        tpr, fpr = true_false_positive(threshold_vector, y_test)
        roc = np.append(roc, [fpr, tpr])
        
    return roc.reshape(-1, 2)

def roc_2(y_true,y_pred):
  #plt.figure(figsize=(15,7))
  ROC = roc_from_scratch(y_pred,y_true,partitions=10)
  plt.scatter(ROC[:,0],ROC[:,1],color='#0F9D58',s=100)
  plt.title('ROC Curve',fontsize=20)
  plt.xlabel('False Positive Rate',fontsize=16)
  plt.ylabel('True Positive Rate',fontsize=16)
def create_triangle(tpr_0,tpr_1,fpr_0,fpr_1):
    plt.plot([tpr_0,tpr_1],[fpr_0,fpr_1], 'k-', lw=2,color='#4285F4')
    plt.plot([tpr_0,tpr_1],[fpr_1,fpr_1], 'k-', lw=2,color='#4285F4')
    plt.plot([tpr_0,tpr_0],[fpr_0,fpr_1], 'k-', lw=2,color='#4285F4')
    #print('area: ',(tpr_0-tpr_1)*(fpr_0-fpr_1)/2)

def create_rectangle(tpr_0,fpr_0):
    plt.plot([tpr_0,tpr_0],[fpr_0,0], 'k-', lw=2,color='#4285F4')


def auc_2(y_pred,y_true):
  for i in range(30):
    ROC = roc_from_scratch(y_pred,y_true, partitions=5 + 2 * i)
    fpr, tpr = ROC[:, 0], ROC[:, 1]
    plt.scatter(fpr, tpr, color="#0F9D58", s=100)
    plt.plot([1, 0], [0, 0], "k-", lw=2, color="#4285F4")
    plt.title("ROC Curve", fontsize=20)
    plt.xlabel("False Positive Rate", fontsize=16)
    plt.ylabel("True Positive Rate", fontsize=16)
    for j in range(5 + 2 * i):
        create_rectangle(fpr[j], tpr[j])
    for k in range(5 + 2 * i):
        create_triangle(fpr[k], fpr[k + 1], tpr[k], tpr[k + 1])
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns 
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve, auc
from sklearn.metrics import precision_recall_curve
import pylab as pl
from sklearn import metrics

def acc_score(df4):
  y_true=df4['Bound']
  y_pred=df4['predicted']
  accuracy_score(y_true, y_pred)
  return y_true, y_pred



def confusion_mat(y_true,y_pred):
  array3=confusion_matrix(y_true, y_pred)

  plt.figure(figsize=(5,5))   
  sns.heatmap(array3, annot=True,fmt='.2f',cmap="PuBu")
  plt.title('Confusion Matrix',fontsize=15,color='red')
  plt.xlabel('Predicted')
  plt.ylabel('True')
  plt.show()


def auc_calculate(df4):
  y_log_score = df4['log odd score']
  precision, recall, thresholds = precision_recall_curve(y_true, y_log_score)
  area = auc(recall, precision)
  print("Area Under Curve")
  print(area)

  pl.clf()
  pl.plot(recall, precision, label='Precision-Recall curve')
  pl.xlabel('Recall')
  pl.ylabel('Precision')
  pl.ylim([0.0, 1.0])
  pl.xlim([0.0, 1.0])
  pl.title('Precision-Recall example: AUC=%0.2f' % area)
  pl.legend(loc="lower left")
  pl.show()

def roc_calculate(df4):
  y_log_score = df4['log odd score']
  fpr,tpr, thresh = metrics.roc_curve(y_true, y_log_score)
  auc = metrics.auc(fpr, tpr)
  print("AUC:", auc)

  plt.plot(fpr, tpr, label='ROC curve (area = %.2f)' %auc)
  plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r', label='Random guess')
  plt.title('ROC curve')
  plt.xlabel('False Positive Rate')
  plt.ylabel('True Positive Rate')
  plt.grid()
  plt.legend()
  plt.show()
%cd /content/drive/MyDrive/BTech Proj/Data/MA0003.4
#df =  motif 
#df2 = non motif
#df4 = mixed test file
df = pd.read_csv("/content/drive/MyDrive/BTech Proj/Data/MA0003.4/motif.txt", delimiter="\t")

df2 = pd.read_csv("/content/drive/MyDrive/BTech Proj/Data/MA0003.4/nonmotif.txt", delimiter="\t")
df1 = list(df["seq"])
df3 = list(df2["seq"])
#df1.reset_index(drop=True, inplace=True)
#df3.reset_index(drop=True, inplace=True)
df4 = pd.read_csv("/content/drive/MyDrive/BTech Proj/Data/MA0003.4/mixed.txt", delimiter="\t")

#df4=df4.rename(columns=df4.iloc[0]).drop(df4.index[0])
#df4 = df4.sample(frac=1).reset_index(drop=True)
df4.head(10)
# cross validation
from sklearn.model_selection import KFold

motif_seq = df['seq'].to_list()
nonmotif_seq = df2['seq'].to_list()

print("Len of motif list: ", len(motif_seq))
print("Len of non motif list: ", len(nonmotif_seq))
X_motifs = motif_seq
y_motifs = [1 for i in range(len(motif_seq))]

X_nonmotifs = nonmotif_seq
y_nonmotifs = [1 for i in range(len(nonmotif_seq))]

X_motifs[0:5]
kf = KFold(n_splits=10)
print(kf.get_n_splits(X_motifs))<jupyter_output>10<jupyter_text>for motif CV<jupyter_code>all_train_index_motif, all_test_index_motif  = [], []
all_train_index_nonmotif, all_test_index_nonmotif  = [], []

#motif
for train_index, test_index in kf.split(X_motifs): 
  #print("TRAIN: ", train_index, "TEST: ", test_index)
  all_train_index_motif.append(train_index)
  all_test_index_motif.append(test_index)

#nonmotif
for train_index, test_index in kf.split(X_nonmotifs): 
  #print("TRAIN: ", train_index, "TEST: ", test_index)
  all_train_index_nonmotif.append(train_index)
  all_test_index_nonmotif.append(test_index)


print(len(all_train_index_motif[0]))
print(len(all_train_index_motif))
print(len(all_train_index_nonmotif))
len(all_train_index_nonmotif[0])
len(all_test_index_nonmotif)
len(all_test_index_motif[0])
X_nonmotifs[:2]

#for i in range(len(all_test_index_nonmotif[0])): 
  print(X_nonmotifs[all_test_index_nonmotif[0][i]])<jupyter_output><empty_output><jupyter_text>list containing all the test motif and non motif sequences<jupyter_code>for i in range(10):
  l_motif=[]
  l_nonmotif=[]

  motif_seq=[]
  for a in all_train_index_motif[i]:
    motif_seq.append(X_motifs[a])

  nonmotif_seq=[]
  for a1 in all_train_index_nonmotif[i]:
    nonmotif_seq.append(X_nonmotifs[a1])
  

  for j in range(len(all_test_index_motif[0])):
    test_set_motif=X_motifs
    l_motif.append(X_motifs[all_test_index_motif[i][j]])

    test_set_nonmotif=X_nonmotifs
    l_nonmotif.append(X_nonmotifs[all_test_index_nonmotif[i][j]])

  l_1 =[1 for i in range(len(l_motif))]
  l_0 =[0 for i in range(len(l_nonmotif))]

  df_test= pd.DataFrame(list(zip(l_motif,l_1)),columns=['seq','Bound'])
  extra={'seq':l_nonmotif,'Bound':l_0}
  df_test= df_test.append(pd.DataFrame(extra))
  df_test=df_test.sample(frac=1).reset_index(drop=True)
  #print(df_test)

  for i1 in range(2,3,1):
    print("Step: ", i1)
    var = i1
    dict6 = train(motif_seq,nonmotif_seq,df_test)
    print("TCT: ",dict6['TCT'])
    print("CTC: ",dict6['CTC'])
    print("TCG ",dict6['TCG'])
    acc = accuracy(df_test)
    print(acc)  
    y_true,y_pred = acc_score(df_test)
    confusion_mat(y_true,y_pred)
    auc_calculate(df_test)
    roc_calculate(df_test)
 len(all_train_index_motif)
for i in range(10):
  ll=[]
  for a in all_train_index_motif[i]:
    ll.append(X_motifs[a])
print(len(ll[0]))
ll=[]
for a in all_train_index_motif[0]:
  ll.append(X_motifs[a])
len(ll)<jupyter_output><empty_output><jupyter_text>list containing all the test nonmotif sequences<jupyter_code>l_nonmotif=[]
for i in range(10):
  for j in range(len(all_test_index_nonmotif[0])):
    test_set_nonmotif=X_nonmotifs
    l_nonmotif.append(X_nonmotifs[all_test_index_nonmotif[i][j]])
print(len(l_nonmotif))
l_1 =[1 for i in range(len(l_motif))]
len(l_1)

l_0 =[0 for i in range(len(l_nonmotif))]
len(l_0)
df_test= pd.DataFrame(list(zip(l_motif,l_1)),columns=['seq','Bound'])
extra={'seq':l_nonmotif,'Bound':l_0}
df_test= df_test.append(pd.DataFrame(extra))
df_test
df_test=df_test.sample(frac=1).reset_index(drop=True)
df_test


for i in range(2,16,1):
  print("Step: ", i)
  var = i
  
  dict6 = train(df1,df3,df4)
  dict6 = sorted(dict6.items(), key=lambda x: x[1], reverse=True)
  print('Printing Dictionary at Step: ', i, )
  for j in range(len(dict6)): 
    max = 0
    if len(dict6[j][0])>=var and max<=10:
      max+=1
      #print("Key: ",dict6[j][0], "Value: ", dict6[j][1])
  acc = accuracy()
  print(acc)

  
  y_true,y_pred = acc_score(df4)
  confusion_mat(y_true,y_pred)
  auc_calculate(df4)
  print("########################################################################################")

df4
df4.dtypes
df4['predicted'].unique()
#from sklearn.metrics import classification_report

#target_names = ['nonmotif', 'motif']
#print(classification_report(y_true, y_pred, target_names=target_names))

#from sklearn.metrics import f1_score
#f1_score(y_true, y_pred, average=None)
#f1_score(y_true, y_pred, average='weighted')
len(dict6[0][0])
#area = auc(recall, precision)
#print("Area Under Curve")
#print(area)



#print(thresh)
l = ['A','C','G','T']
for i in range(20):
  ans = random.choice(l)
  print(ans)
def non_motif_string():
    """
    generates a string of length (by get_num) of ACGT
    returns: string 
    """
    length = 314
    # print("random length: " , length)
    #len4 = length / 4
    # print("len / 4 :" , len4)
    letters = ["A", "C", "G", "T"]
    nonmotiflist = []
    for i in range(length): 
      nonmotiflist.append(random.choice(letters))
    # print(" Random generated string with repetition:")
    str1 = ""
    #print(str1.join(nonmotiflist))
    return str1.join(nonmotiflist)

non = non_motif_string()
print(non)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15346088350195333, 'fraction_numerical': 0.02919963938695783, 'mean_word_length': 4.373250807319699, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 44, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '112054', 'license': 'MIT', 'n_tokens_mistral': 8528, 'n_tokens_neox': 8065, 'n_words': 1552, 'path': 'colab_files/full_pipeline_zip_files_13_01_22.ipynb', 'repo_name': 'radhikasethi2011/btechproj', 'size': '19966'}