<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
from google.colab import drive
drive.mount('/content/drive')

import numpy as np

import pandas as pd


data = pd.read_csv('/content/drive/My Drive/Colab Notebooks/Machine/Combined_News_DJIA.csv')
data
date=data.iloc[:,0]
date
Y=data.iloc[:,1]
data

data.shape
b=data.shape[1]
b
data2=data.drop(['Date'],axis=1)
data2=data2.drop(['Label'],axis=1)

print(data2.shape)
outside_list=[]
for i in range(0,5):
        inside_list=[]
        inside_list.append(i)
        inside_list.append(i+1)
        outside_list.append(inside_list)

       #you can access any inside_list from the outside_list and append     
outside_list[1]
print(outside_list)
p=pd.read_csv('/content/drive/My Drive/Colab Notebooks/Machine/positive-word-list.txt')
p1=p.to_numpy()
p1=p1.reshape(1,-1)
pattern=p1
pattern
p2=pattern.tolist()
p3=p2[0]
print(p3)
listnew=['impeach','welcome','adorable']
p3.append(listnew)


data2
def function(words):
        count=0
        for word in words:
                if word in p3: 
                 count=count +1
        return count

X2=data2.iloc[1:2,1]
print(X2)

X2=X2.to_string() 


import string
X2=data2.iloc[3:4,1]
print(X2)
translator = str.maketrans('', '', string.punctuation)
X2=X2.to_string() 
X2=(X2.translate(translator))
X2.lower()
List=X2.split()
print(List)
len(List)
function(X2)
translator = str.maketrans('', '', string.punctuation)
translator = str.maketrans('', '', string.punctuation)
import string
X2=data2.iloc[3:4,1]
X2
X2.to_string
import string
X2=data2.iloc[3:4,1]
print(X2)
translator = str.maketrans('', '', string.punctuation)
X2=X2.to_string() 
X2.translate(translator)
X2
X2.split()
list1=[]
translator = str.maketrans('', '', string.punctuation)
for i in range(25):
   insidelist=[]
   for j in range(1989):
         X2=data2.iloc[j:j+1,i]
         X2=X2.to_string() 
         X2.lower()
         X2.translate(translator)
         X3=X2.split()
         y1=function(X3)
         insidelist.append(y1)
   list1.append(insidelist)
list1
Y=pd.DataFrame(list1)
Y
 arr = Y.to_numpy()

print('\nNumpy Array\n----------\n', arr)
new=arr.transpose()
new
DataT=pd.DataFrame(new)
DataT
from google.colab import files
DataT.to_csv('filename.csv') 
files.download('filename.csv')


DataT.to_csv('mydataframe2.csv')
files.download('mydataframe2.csv')
dataT=pd.read_csv('/content/drive/My Drive/Colab Notebooks/Machine/mydataframe3.csv')
dataT

import keras
import keras.utils
from keras import utils as np_utils

keras.preprocessing.text.Tokenizer(num_words=None, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ', char_level=False, oov_token=None, document_count=0)
import sklearn
import tensorflow as tf
import numpy as np
import pandas as pd
import os
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt


import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()


import tensorflow as tf
import tensorflow 

from tensorflow import keras
from keras.layers import Dense

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
keras.preprocessing.text.Tokenizer(num_words=None, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ', char_level=False, oov_token=None, document_count=0)<jupyter_output><empty_output><jupyter_text><jupyter_code>from keras.preprocessing.text import Tokenizer
# define 5 documents
docs = ['Well done!',
		'Good work',
		'Great effort',
		'nice work',
		'Excellent!']
# create the tokenizer
t = Tokenizer(num_words=100)
# fit the tokenizer on the documents
t.fit_on_texts(docs)
print(t.word_counts)
print(t.document_count)
print(t.word_index)
print(t.word_docs)
text='I love you'
X_seq2=keras.preprocessing.text.one_hot(text, 100, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ')
X_seq2<jupyter_output><empty_output><jupyter_text><jupyter_code>text='I love you '
keras.preprocessing.text.text_to_word_sequence(text, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ')
import string
translator = str.maketrans('', '', string.punctuation)
X2=data2.iloc[0,0]
X2
X2.lower()
X2.translate(translator)

translator = str.maketrans('', '', string.punctuation)
insidelist=[]
j=0
for i in range(25):
   X2=data2.iloc[0,i]
   X2.lower()
   X2.translate(translator)
   keras.preprocessing.text.text_to_word_sequence(X2, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ')
   X3=keras.preprocessing.text.one_hot(X2, 100, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ')
   insidelist.append(X3)
insidelist
sum(insidelist, [])



list1=[]
translator = str.maketrans('', '', string.punctuation)
insidelist=[]
j=0
X2=data2.iloc[0,1]
X2.lower()
print(X2)
X2.translate(translator)
print(X2)
X3=X2.split()
print(X3)
insidelist.append(X3)
insidelist



list1=[]
translator = str.maketrans('', '', string.punctuation)
for j in range(200):
  insidelist=[]
  for i in range(25):
      X2=data2.iloc[j,i]
      X2.lower()
      X2.translate(translator)
      keras.preprocessing.text.text_to_word_sequence(X2, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ')
      X3=keras.preprocessing.text.one_hot(X2, 100, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ')
      insidelist.append(X3)

  p=sum(insidelist,[])
  list1.append(p)
list1
Z=np.array(list1)

translator = str.maketrans('', '', string.punctuation)
insidelist=[]
j=1988
for i in range(25):
   X2=data2.iloc[j,i]
   X2.lower()
   X2.translate(translator)
   keras.preprocessing.text.text_to_word_sequence(X2, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ')
   X3=keras.preprocessing.text.one_hot(X2, 100, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ')
   insidelist.append(X3)
insidelist
sum(insidelist, [])


list1=[]
import string
translator = str.maketrans('', '', string.punctuation)
for j in range(1989):
 insidelist=[]
 for i in range(25):
      X2=data2.iloc[j,i]
      if type(X2)==str:
        X2.translate(translator)
        keras.preprocessing.text.text_to_word_sequence(X2, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ')
        X3=keras.preprocessing.text.one_hot(X2, 90, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, split=' ')
        insidelist.append(X3)
      else:
        print(i,j,X2)  

 p=sum(insidelist,[])
 list1.append(p)
list1
DF=pd.DataFrame(list1)
DF
DF2=DF.fillna(0)
DF2.values
X_train_full=DF2.values
X_train_full.shape
#Exporting the main data to csv file to prevent loss of this data during tensor flow downgrade

DF2.to_csv('mydataframe4.csv')
files.download('mydataframe4.csv')
from sklearn.model_selection import train_test_split
Y_train_full=data.iloc[:,1]
X_train, X_test, y_train, y_test = train_test_split( X_train_full, Y_train_full, test_size=0.10, random_state=42) 
X_train
X_test
y_train
X_valid, X_train = X_train[:100] , X_train[100:] 
y_valid, y_train = y_train[:100], y_train[100:]
X_valid



!pip install tensorflow==1.14

def LRN2D(x):
    import tensorflow as tf
    return tf.nn.lrn(x, alpha=1e-4, beta=0.75)




import tensorflow as tf
# Create a tf.keras model.
model = tf.keras.Sequential()
model.add(tf.keras.layers.Dense(1, input_shape=[10]))
model.summary()
from keras import Sequential



from keras.layers import Embedding, LSTM, Dense, Dropout

tf.compat.v1.get_default_graph()

NB_WORDS=7000
from keras import Sequential
from keras.layers import Dense, Embedding, Dropout, LSTM
# model parameters:
EMBED_DIM = 50
LSTM_UNITS = 32
MAXLEN=1000

model = Sequential()

model.add(Embedding(NB_WORDS, EMBED_DIM, input_length=MAXLEN))
model.add(Dropout(0.2))
model.add(LSTM(LSTM_UNITS))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
print(model.summary())


num_epochs=5
model.compile(loss='binary_crossentropy', 
             optimizer='adam', 
             metrics=['accuracy'])
model.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=num_epochs)
import pandas as pd
Xtrain=pd.read_csv('/content/drive/My Drive/Colab Notebooks/Machine/mydataframe5.csv')
data = pd.read_csv('/content/drive/My Drive/Colab Notebooks/Machine/Combined_News_DJIA.csv')
X_train_full=Xtrain.values
X_train_full.shape

from sklearn.model_selection import train_test_split
Y_train_full=data.iloc[:,1]
X_train, X_test, y_train, y_test = train_test_split( X_train_full, Y_train_full, test_size=0.10, random_state=42) 
X_train
X_test
y_train
X_valid, X_train = X_train[:100] , X_train[100:] 
y_valid, y_train = y_train[:100], y_train[100:]
X_valid

from keras.layers import Embedding, LSTM, Dense, Dropout
embedding_size=32
vocabulary_size=500
max_words=749
model=Sequential()
model.add(Embedding(vocabulary_size, embedding_size, input_length=max_words))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
print(model.summary())

num_epochs=5
model.compile(loss='binary_crossentropy', 
             optimizer='adam', 
             metrics=['accuracy'])
model.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=num_epochs)
scores = model.evaluate(X_test, y_test, verbose=0)
scores
print('Test accuracy:', scores[1])
import matplotlib.pyplot as plt


history = model.fit(X_train, y_train, batch_size=128,
          epochs=num_epochs)
plt.figure(figsize=(5,3))
plt.plot(history.epoch,history.history['loss'], label='training')
plt.title('loss')
plt.legend(loc='best')

plt.figure(figsize=(5,3))
plt.plot(history.epoch,history.history['acc'], label='training')
plt.plot(history.epoch,history.history['acc'], label='validation')
plt.title('accuracy')
plt.legend(loc='best');
import tensorflow as tf
# Create a tf.keras model.
model = tf.keras.Sequential()
model.add(tf.keras.layers.Dense(1, input_shape=[10]))
model.summary()
from keras import Sequential
from keras.layers import Embedding, LSTM, Dense, Dropout
embedding_size=64
vocabulary_size=800
max_words=749
model=Sequential()
model.add(Embedding(vocabulary_size, embedding_size, input_length=max_words))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
print(model.summary())

from keras.layers import Embedding, LSTM, Dense, Dropout
embedding_size=64
vocabulary_size=500
max_words=749
model=Sequential()
model.add(Embedding(vocabulary_size, embedding_size, input_length=max_words))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
print(model.summary())
num_epochs=10
model.compile(loss='binary_crossentropy', 
             optimizer='adam', 
             metrics=['accuracy'])
model.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=num_epochs)
history2 = model.fit(X_train, y_train, batch_size=128,
          epochs=num_epochs)
plt.figure(figsize=(5,3))
plt.plot(history2.epoch,history2.history['loss'], label='training')
plt.title('loss')
plt.legend(loc='best')

plt.figure(figsize=(5,3))
plt.plot(history2.epoch,history2.history['acc'], label='training')
plt.plot(history2.epoch,history2.history['acc'], label='validation')
plt.title('accuracy')
plt.legend(loc='best');
import seaborn as sns
sns.set(style="whitegrid")
ax = sns.boxplot(y=history.history['acc'])
q=pd.read_csv('/content/drive/My Drive/Colab Notebooks/Machine/negative-word-list.txt')

q1=q.to_numpy()

q1=q1.reshape(1,-1)
pattern=q1
q2=pattern.tolist()
q3=q2[0]
#there is extra space so need to strip that
q3 = [x.strip(' ') for x in q3]
print(q3)
def function(words):
          count=0
          for word in words:
              if word in p3: 
                count=count +1
            
          return count

def something(words):
          count2=0
          for word in words:
            if word in q3:
                count2=count2+1
          count3= function(words)-count2
          return count3

#Kept the same intendation so that function can call the other function. This function will find difference between positive words and negative words<jupyter_output><empty_output><jupyter_text><jupyter_code>something(['nice','loving','great','adorable','abused'])
#Positive and Negative words analysis oper news paper article


data.shape
b=data.shape[1]
b
data2=data.drop(['Date'],axis=1)
data2=data2.drop(['Label'],axis=1)

print(data2.shape)
import string
list2=[]
translator = str.maketrans('', '', string.punctuation)
for i in range(25):
   insidelist=[]
   for j in range(1989):
         X2=data2.iloc[j:j+1,i]
         X2=X2.to_string() 
         X2.lower()
         X2.translate(translator)
         X3=X2.split()
         y1=something(X3)
         insidelist.append(y1)
   list2.append(insidelist)
list2
Y2=pd.DataFrame(list2)
Y2
y3=Y2.values
y4=y3.transpose()
DF3=pd.DataFrame(y4)
DF3.shape
X_train_full2=DF3

X_train_full2.to_csv('mydataframesentiv1.csv') 
files.download('mydataframesentiv1.csv')
import pandas as pd


datasenti = pd.read_csv('/content/drive/My Drive/Colab Notebooks/Machine/mydataframesentiv2.csv')
datasenti

X_train_full2=datasenti

from sklearn.model_selection import train_test_split
Y_train_full2=data.iloc[:,1]
X_train2, X_test2, y_train2, y_test2 = train_test_split( X_train_full2, Y_train_full2, test_size=0.10, random_state=42) 
X_train2
X_test2
y_train2

from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

from matplotlib import pyplot as plt

#Training on Model 
lda = LinearDiscriminantAnalysis(store_covariance=True)
model1=lda.fit(X_train2, y_train2)
lda.fit
## Quadratic Discriminant Analysis
qda = QuadraticDiscriminantAnalysis(store_covariance=True)
model2=qda.fit(X_train2, y_train2)

log = LogisticRegression()
modellog=log.fit(X_train2,y_train2)
    
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix
ypred1=modellog.predict(X_test2)
ypred1
conf_mx_1 = confusion_matrix(y_test2, ypred1)
print(conf_mx_1)
class_names=[0,1]
#Confusion matrix represents for 4 levels 0,1,2,3
accuracy100=(conf_mx_1[0,0]+conf_mx_1[1,1])/(np.sum(conf_mx_1))
accuracy100
print("Accuracy for Model3 Logistic ",accuracy100)
title = [("Confusion matrix Logistic", None)]
disp = plot_confusion_matrix(model1, X_test2, y_test2,
                                 display_labels=class_names,
                                 cmap=plt.cm.Blues)
                                
disp.ax_.set_title(title)

print(title)
print(disp.confusion_matrix)

plt.show()
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix
ypred1=model1.predict(X_test2)
ypred1
conf_mx_1 = confusion_matrix(y_test2, ypred1)
print(conf_mx_1)
class_names=[0,1]
#Confusion matrix represents for 4 levels 0,1,2,3
accuracy100=(conf_mx_1[0,0]+conf_mx_1[1,1])/(np.sum(conf_mx_1))
accuracy100
print("Accuracy for Model1 LDA ",accuracy100)
title = [("Confusion matrix LDA", None)]
disp = plot_confusion_matrix(model1, X_test2, y_test2,
                                 display_labels=class_names,
                                 cmap=plt.cm.Blues)
                                
disp.ax_.set_title(title)

print(title)
print(disp.confusion_matrix)

plt.show()

ypred1=model2.predict(X_test2)
ypred1
conf_mx_1 = confusion_matrix(y_test2, ypred1)
print(conf_mx_1)
class_names=[0,1]
accuracy200=(conf_mx_1[0,0]+conf_mx_1[1,1])/(np.sum(conf_mx_1))
accuracy200
print("Accuracy for Model1 QDA ",accuracy200)
title = [("Confusion matriX QDA", None)]
disp = plot_confusion_matrix(model2, X_test2, y_test2,
                                 display_labels=class_names,
                                 cmap=plt.cm.Blues)
                                
disp.ax_.set_title(title)

print(title)
print(disp.confusion_matrix)

plt.show()

#Now using Financial Data for further analysis 
pd.read_csv('/content/drive/My Drive/Colab Notebooks/Machine/

import datetime

x = datetime.datetime(2018, 9, 15, 12, 45, 35)

print(x.strftime("%b %d %Y %H:%M:%S"))
# there are more observations in SP stock and hence we need to extract data where we have Redit News
date
dateDF=pd.DataFrame(date)

DFdate=pd.concat([dateDF,X_train_full2], axis=1, sort=False)
DFdate
DFdate.columns
DFdate.columns.str.replace(' ','')
DFdate.columns

DFdate

DFdate

date=data.iloc[:,0]
date
pd.merge(DFdate,SPstock[['Date','Open','High','Low','Close']],on='Date',how='inner',left_index=True)
spstock=pd.read_csv('/content/drive/My Drive/Colab Notebooks/Machine/vxocurrent.csv')
spstock.columns=spstock.columns.str.replace(' ','')
import datetime
spstock['Date'] = pd.to_datetime(spstock.Date).dt.strftime('%Y-%m-%d')
spstock['Date']
SPnew=spstock.iloc[:,1:5]
dateDF=pd.DataFrame(spstock['Date'])
SPstocknew=pd.concat([dateDF,SPnew],axis=1,sort=False)
SPstocknew
#Merging SP stock with Sentimental analysis data frame
DFsentFY=pd.merge(DFdate,SPstocknew,how='inner',on='Date')
DFsentFY
DFsenFY2=DFsentFY.iloc[:,1:30]
X_train_full3=DFsenFY2
X_train_full3

from sklearn.model_selection import train_test_split
Y_train_full3=data.iloc[:,1]
X_train3, X_test3, y_train3, y_test3 = train_test_split( X_train_full3, Y_train_full3, test_size=0.10, random_state=42) 
X_train3
X_test3
y_train3


from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

from matplotlib import pyplot as plt

#Training on Model 
lda = LinearDiscriminantAnalysis(store_covariance=True)
model1=lda.fit(X_train3, y_train3)
lda.fit
## Quadratic Discriminant Analysis
qda = QuadraticDiscriminantAnalysis(store_covariance=True)
model2=qda.fit(X_train3, y_train3)
## Logisitic Regression
clf = LogisticRegression()
model3=clf.fit(X_train3, y_train3)
clf.fit


print("Score LDA All predictors : %.3f"%lda.score(X_train3,y_train3))
print("Score QDA All predictors : %.3f"%qda.score(X_train3,y_train3))
print("Score Logistic All predictors : %.3f"%clf.score(X_train3,y_train3))

from sklearn.metrics import confusion_matrix
ypred3=model1.predict(X_test3)
ypred3
conf_mx_2 = confusion_matrix(y_test3, ypred3)
print(conf_mx_2)
#Confusion matrix represents for 4 levels 0,1,2,3
accuracy200=(conf_mx_2[0,0]+conf_mx_2[1,1])/(np.sum(conf_mx_2))
accuracy200
print("Accuracy for Model1 LDA ",accuracy200)
ypred33=model1.predict(X_test3)
conf_mx_3 = confusion_matrix(y_test3, ypred33)
print(conf_mx_3)
#Confusion matrix represents for 4 levels 0,1,2,3
accuracy300=(conf_mx_3[0,0]+conf_mx_3[1,1])/(np.sum(conf_mx_3))
print("Accuracy for Model2 QDA ",accuracy300)

ypred44=model1.predict(X_test3)
conf_mx_4 = confusion_matrix(y_test3, ypred44)
print(conf_mx_4)
#Confusion matrix represents for 4 levels 0,1,2,3
accuracy400=(conf_mx_4[0,0]+conf_mx_4[1,1])/(np.sum(conf_mx_4))
print("Accuracy for Model2 Logistic ",accuracy400)
objects=("LDA","QDA","Logistic")
Accuracy=[accuracy200*100,accuracy300*100,accuracy400*100]
y_pos=np.arange(len(objects))
plt.bar(y_pos,Accuracy,align='center',alpha=0.5)
plt.ylabel('Accuracy%')
plt.xticks(y_pos,objects)
plt.title("Accuracy on Test Data  with all Predictors for various models")

from pandas import DataFrame
from pandas import concat
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
# create sequence
length = 10
sequence = [i/float(length) for i in range(length)]
# create X/y pairs
df = DataFrame(sequence)

df = concat([df, df.shift(1)], axis=1)
df.dropna(inplace=True)
# convert to LSTM friendly format
values = df.values
X, y = values[:, 0], values[:, 1]
print(X)
X = X.reshape(-1,1)
print(X)
p=X.shape(1)
print(p)

from pandas import DataFrame
from pandas import concat
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
# create sequence
length = 10
sequence = [i/float(length) for i in range(length)]
# create X/y pairs
df = DataFrame(sequence)
print(df)
print(df.shift(1))
dfnew = concat([df, df.shift(1)], axis=1)
print(dfnew)
dfnew.dropna(inplace=True)
print(dfnew)

from keras.preprocessing.sequence import TimeseriesGenerator
import numpy as np

data = np.array([[i] for i in range(50)])
targets = np.array([[i] for i in range(50)])

data_gen = TimeseriesGenerator(data, targets,
                               length=10, sampling_rate=2,
                               batch_size=2)
assert len(data_gen) == 20
print(data_gen)
batch_0 = data_gen[0]
x, y = batch_0
assert np.array_equal(x,
                      np.array([[[0], [2], [4], [6], [8]],
                                [[1], [3], [5], [7], [9]]]))
assert np.array_equal(y,
                      np.array([[10], [11]]))
#Social Sentimental Analysis 
X_train_full3.shape[0]
p=[]
z=[]
outersum=[]
for j in range((X_train_full3.shape[0])):
   z=(X_train_full3.iloc[j,:25])
   p.append(z)
   sum=0
   innersum=[]
   x=p[j]
   for i in range(25):
      sum=x[i]+sum
   innersum.append(sum)
   outersum.append(innersum)
df = pd.DataFrame.from_records(outersum)
df

SPstocknewsenti=pd.concat([X_train_full3,df],axis=1,sort=False)
print(SPstocknewsenti.columns)
SPstocknewsenti.rename(columns={0: "Sentimentalscore"})
reformedDF=SPstocknewsenti.iloc[:,25:]
reformedDF
df10=reformedDF.rename(columns={0:"Sentimentalscore"})
df10

df10.to_csv('Financialsenti.csv') 
files.download('Financialsenti.csv')
dataFinal = pd.read_csv('/content/drive/My Drive/Colab Notebooks/Machine/Financialsentiv.csv')
dataFinal<jupyter_output>0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
{'fraction_non_alphanumeric': 0.15021740146804433, 'fraction_numerical': 0.037495909112160455, 'mean_word_length': 5.137733142037303, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '562627', 'license': 'Apache-2.0', 'n_tokens_mistral': 8607, 'n_tokens_neox': 8178, 'n_words': 1489, 'path': 'Copy_of_Textanalytics.ipynb', 'repo_name': 'pranshu20p/colabtools', 'size': '21389'}