<jupyter_start><jupyter_text>**Information Warfare**By <NAME> III<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sb




import collections

import tweepy as tw
import nltk
from nltk.corpus import stopwords
from textblob import TextBlob

import warnings
warnings.filterwarnings("ignore")
url1 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_1.csv'
url2 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_2.csv'
url3 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_3.csv'
url4 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_4.csv'
url5 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_5.csv'
url6 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_6.csv'
url7 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_7.csv'
url8 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_8.csv'
url9 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_9.csv'
url10 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_10.csv'
url11 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_11.csv'
url12 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_12.csv'
url13 = 'https://raw.githubusercontent.com/fivethirtyeight/russian-troll-tweets/master/IRAhandle_tweets_13.csv'

#datasource: https://www.tandfonline.com/doi/abs/10.1080/10584609.2020.1718257
#<NAME> & <NAME> (2020) Troll Factories: Manufacturing Specialized Disinformation on Twitter, Political Communication, 37:4, 447-467, DOI: 10.1080/10584609.2020.1718257

df1 = pd.read_csv(url1,index_col=0,parse_dates=[0]) #read csv data files
df2 = pd.read_csv(url2,index_col=0,parse_dates=[0])
df3 = pd.read_csv(url3,index_col=0,parse_dates=[0])
df4 = pd.read_csv(url4,index_col=0,parse_dates=[0])
df5 = pd.read_csv(url5,index_col=0,parse_dates=[0])
df6 = pd.read_csv(url6,index_col=0,parse_dates=[0])
df7 = pd.read_csv(url7,index_col=0,parse_dates=[0])
df8 = pd.read_csv(url8,index_col=0,parse_dates=[0])
df9 = pd.read_csv(url9,index_col=0,parse_dates=[0])
df10 = pd.read_csv(url10,index_col=0,parse_dates=[0])
df11 = pd.read_csv(url11,index_col=0,parse_dates=[0])
df12 = pd.read_csv(url12,index_col=0,parse_dates=[0])
df13 = pd.read_csv(url13,index_col=0,parse_dates=[0])
df14 = df1.append(df2)    #append all the datasets into a single dataframe
df15 = df14.append(df3)
df16 = df15.append(df4)
df17 = df16.append(df5)
df18 = df17.append(df6)
df19 = df18.append(df7)
df20 = df19.append(df9)
df21 = df20.append(df10)
df22 = df21.append(df11)
df23 = df22.append(df12)
df24 = df23.append(df13)
df24.info() #review datatypes and column names

df24.head() #review a sample fo the data
df24.isnull().sum()
# this tells me which columns have null values and how many<jupyter_output><empty_output><jupyter_text>**Notes on Null Values from Data Dictionary**1.   region - A region classification, as determined by Social Studio2.   post_type	- Indicates if the tweet was a retweet or a quote-tweet3.   tco_step1 - tco1_step1	First redirect for the first http(s)://t.co/ link in a tweet, if it existstco2_step1	First redirect for the second http(s)://t.co/ link in a tweet, if it existstco3_step1	First redirect for the third http(s)://t.co/ link in a tweet, if it exists.<jupyter_code># Now I'm just going to drop things that are obviously usless but don't nessecarily have null values
df24.drop(['account_type','account_category','article_url'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis<jupyter_code>df24
sb.set_context({"figure.figsize": (10, 5)})
plot = sb.countplot(df24['author'], order=df24.author.value_counts().iloc[:8].index)
plt.xticks(rotation=55)
import string
string.punctuation

df24
df24['language'].value_counts()
df24[df24["language"].str.contains("English")==True] #drops all non-english tweets
!pip install tweet-preprocessor
import preprocessor as p
# https://www.kaggle.com/sreejiths0/efficient-tweet-preprocessing
train_df = df24.dropna()
train_df = df24.drop_duplicates()
train_df.rename(columns = {'content' : 'text'}, inplace = True)
def preprocess_tweet(row):
    text = row['text']
    text = p.clean(text)
    return text
train_df['text']=train_df['text'].apply(str)
train_df['text'] = train_df.apply(preprocess_tweet, axis=1) 

#Tweets have been cleaned to normal text.
#Lowercasing
#Punctuation Removal
#Replace extra white spaces
#Stopwords removal
#For stop word removal , i have used gensim library

train_df.head()

from gensim.parsing.preprocessing import remove_stopwords

def stopword_removal(row):
    text = row['text']
    text = remove_stopwords(text)
    return text
train_df['text'] = train_df.apply(stopword_removal, axis=1) # remove stopwords

train_df.head()
train_df['text'] = train_df['text'].str.lower().str.replace('[^\w\s]',' ').str.replace('\s\s+', ' ')
train_df.head()


from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt
stopwords = set(STOPWORDS)

def show_wordcloud(data, title = None):
    wordcloud = WordCloud(
        background_color='white',
        stopwords=stopwords,
        max_words=200,
        max_font_size=40, 
        scale=3,
        random_state=1 # chosen at random by flipping a coin; it was heads
    ).generate(str(data))

    fig = plt.figure(1, figsize=(12, 12))
    plt.axis('off')
    if title: 
        fig.suptitle(title, fontsize=20)
        fig.subplots_adjust(top=2.3)

    plt.imshow(wordcloud)
    plt.show()

show_wordcloud(train_df['text'])
corpus=[]
new= train_df['text'].str.split()
new=new.values.tolist()
corpus=[word for i in new for word in i] #creates a corpus of twitter words
corpus 
data_analysis = nltk.FreqDist(corpus)
data_analysis.plot(50, cumulative=False) #seems like there are stop words in here
train_df.info()
from nltk.corpus import stopwords
nltk.download('stopwords')
stop_words = set(stopwords.words("english"))
filtered_corpus =[]
for w in corpus:
  if w not in stop_words:
    filtered_corpus.append(w)
plt.figure(figsize=(10,5))
data_analysis = nltk.FreqDist(filtered_corpus)
data_analysis.plot(20,cumulative=False) #seems like there are stop words in here
# this looks better to me, fewer words that seem like stop words
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
res = ' '.join(map(str, filtered_corpus))
wordcloud = WordCloud().generate(res)
#interestingly, the third most common word is police
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()
from nltk.util import ngrams
from sklearn.feature_extraction.text import CountVectorizer

#people tend to believe what their social network is telling them is true, "friend", and "believe" standout here, mention research into villages and birthcontrol from viral spread of info book
import collections
def get_top_ngram(corp, n=None):
    vec = CountVectorizer(ngram_range=(n, n)).fit(corp)
    bag_of_words = vec.transform(corp)
    sum_words = bag_of_words.sum(axis=0) 
    words_freq = [(word, sum_words[0, idx]) 
                  for word, idx in vec.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
    return words_freq[:20]
sb.set_context({"figure.figsize": (5, 10)})
top_n_bigrams=get_top_ngram(train_df['text'],2)[:20]
x,y=map(list,zip(*top_n_bigrams))
sb.barplot(x=y,y=x,)    #bigram
#trigram
top_tri_grams=get_top_ngram(train_df['text'],n=3)
x,y=map(list,zip(*top_tri_grams))
sb.barplot(x=y,y=x)
#https://paperswithcode.com/dataset/liar
#starting the ML, reading in and analyzing the LIAR dataset
! pip install fsspec
liarurl= 'https://raw.githubusercontent.com/ekagra-ranjan/fake-news-detection-LIAR-pytorch/master/train.tsv'
df_liar = pd.read_csv(liarurl, sep='\t', error_bad_lines=False, names=["jsonid", "rating", "text", "metatag", 'person', 'personcat','state', 'party', '1', '2', '3', '4', '5', 'source'], index_col=0)
df_liar.head()
#LIAR: A BENCHMARK DATASET FOR FAKE NEWS DETECTION

#<NAME>, "Liar, Liar Pants on Fire": A New Benchmark Dataset for Fake News Detection, to appear in Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (ACL 2017), short paper, Vancouver, BC, Canada, July 30-August 4, ACL.
#=====================================================================
#Description of the TSV format:

#Column 1: the ID of the statement ([ID].json).
#Column 2: the label.
#Column 3: the statement.
#Column 4: the subject(s).
#Column 5: the speaker.
#Column 6: the speaker's job title.
#Column 7: the state info.
#Column 8: the party affiliation.
#Column 9-13: the total credit history count, including the current statement.
#9: barely true counts.
#10: false counts.
#11: half true counts.
#12: mostly true counts.
#13: pants on fire counts.
#Column 14: the context (venue / location of the speech or statement).

#Note that we do not provide the full-text verdict report in this current version of the dataset,
#but you can use the following command to access the full verdict report and links to the source documents:
#wget http://www.politifact.com//api/v/2/statement/[ID]/?format=json

#======================================================================
#The original sources retain the copyright of the data.

#Note that there are absolutely no guarantees with this data,
#and we provide this dataset "as is",
#but you are welcome to report the issues of the preliminary version
#of this data.

#You are allowed to use this dataset for research purposes only.

#For more question about the dataset, please contact:
#<NAME>, <EMAIL>

#v1.0 04/23/2017


# i dont know what columns 1-5 actually represent, need to figure that our
df_rsntwts = train_df

df_liar['rating'].value_counts()
df_liar.head()
from sklearn.preprocessing import LabelEncoder
lb_make = LabelEncoder()
df_liar['ratingnum'] = lb_make.fit_transform(df_liar['rating'])

df_liar['text'] = df_liar.apply(stopword_removal, axis=1) # remove stopwords
#train_df['text']=train_df['text'].apply(str)
df_liar['text'] = df_liar.apply(preprocess_tweet, axis=1) 
#make corpus of liar text
df_liar.head()
import string

df_liar["text_clean"] = df_liar['text'].str.replace('[^\w\s]','')
df_liar.head()
df_liar["text_clean"] = df_liar["text_clean"].str.lower()
df_liar.head()

from sklearn.model_selection import train_test_split
X = df_liar['text_clean']
y = df_liar['ratingnum']
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = 42, test_size = 0.3)
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer()
X_train_final = vectorizer.fit_transform(X_train)
from sklearn.svm import LinearSVC, SVC
from sklearn.pipeline import Pipeline
text_clf = Pipeline([('tfidf', TfidfVectorizer()),
                      ('clf', LinearSVC()),
                    ])
text_clf.fit(X_train, y_train)
predictions = text_clf.predict(X_test)
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
print(accuracy_score(y_test, predictions))
print(confusion_matrix(y_test, predictions))
print(classification_report(y_test, predictions))
# make everything < mostly true = false
df_liar['ratingnum'].value_counts()
df_liar['rating'].value_counts()
df_liar['ratingnum'].mask(df_liar['ratingnum'] == 3, 5, inplace=True) # change mostly-true to true

df_liar['ratingnum'].mask(df_liar['ratingnum'] == 2, 1, inplace=True)
df_liar['ratingnum'].mask(df_liar['ratingnum'] == 4, 1, inplace=True)
df_liar['ratingnum'].mask(df_liar['ratingnum'] == 0, 1, inplace=True) #change anything <mostly-true to false
# key: 5 = true, 1 = false
df_liar.head()
df_liar['ratingnum'].value_counts()
X = df_liar['text_clean']
y = df_liar['ratingnum']
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = 42, test_size = 0.3)
vectorizer = TfidfVectorizer()
X_train_final = vectorizer.fit_transform(X_train)
#LinearSVC_classifier = SklearnClassifier(SVC(kernel='linear',probability=True)), LinearSVC()
from nltk.classify.scikitlearn import SklearnClassifier
text_clf = Pipeline([('tfidf', TfidfVectorizer()),
                      ('clf', LinearSVC()),
                    ])
text_clf.fit(X_train, y_train)
predictions = text_clf.predict(X_test)
print(accuracy_score(y_test, predictions))
print(confusion_matrix(y_test, predictions))
print(classification_report(y_test, predictions))
#prediction is 61% accurate for detecting truth vs lie within LIAR dataset
#https://medium.com/@hritikattri10/feature-extraction-using-tf-idf-algorithm-44eedb37305e
df_rsntwts.head()
df_rsntwts['ratingnum'] = 1
df_rsntwts.head()
X1 = df_rsntwts['text']
Y1 = df_rsntwts['ratingnum']
predictions = text_clf.predict(X1)
print(accuracy_score(Y1, predictions))
print(confusion_matrix(Y1, predictions))
print(classification_report(Y1, predictions))#predict if IRA tweets are false using the LIAR dataset assuming all IRA tweets are 'false'
! pip install xgboost
from xgboost import XGBClassifier
model = XGBClassifier()

text_xgb = Pipeline([('tfidf', TfidfVectorizer()),
                      ('xgb', model),
                    ])
text_xgb.fit(X_train, y_train)
predictions = text_xgb.predict(X1)

print(accuracy_score(Y1, predictions))
print(confusion_matrix(Y1, predictions))
print(classification_report(Y1, predictions))#predict if IRA tweets are false using the LIAR dataset assuming all IRA tweets are 'false'
from sklearn.neighbors import KNeighborsClassifier
model = KNeighborsClassifier()

text_knn = Pipeline([('tfidf', TfidfVectorizer()),
                      ('knn', model),
                    ])
text_knn.fit(X_train, y_train)
predictions = text_knn.predict(X1)
print(accuracy_score(Y1, predictions))
print(confusion_matrix(Y1, predictions))
print(classification_report(Y1, predictions))#predict if IRA tweets are false using the LIAR dataset assuming all IRA tweets are 'false'
from sklearn.ensemble import RandomForestClassifier
model = RandomForestClassifier()

text_rf = Pipeline([('tfidf', TfidfVectorizer()),
                      ('rf', model),
                    ], verbose=True)
text_rf.fit(X_train, y_train)
predictions = text_rf.predict(X1)
print(accuracy_score(Y1, predictions))
print(confusion_matrix(Y1, predictions))
print(classification_report(Y1, predictions))#predict if IRA tweets are false using the LIAR dataset assuming all IRA tweets are 'false'
from sklearn.naive_bayes import MultinomialNB
model = MultinomialNB()

text_nb = Pipeline([('tfidf', TfidfVectorizer()),
                      ('nb', model),
                    ], verbose=True)
text_nb.fit(X_train, y_train)
predictions = text_nb.predict(X1)
print(accuracy_score(Y1, predictions))
print(confusion_matrix(Y1, predictions))
print(classification_report(Y1, predictions))#predict if IRA tweets are false using the LIAR dataset assuming all IRA tweets are 'false'
import nltk
nltk.download('vader_lexicon')
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()
df_rsntwts['scores'] = df_rsntwts['text'].apply(lambda text: sid.polarity_scores(text))
df = df_rsntwts

df['compound'] = df['scores'].apply(lambda score_dict: score_dict['compound'])
df['sentiment_type']=''
df['compound']
df.loc[df.compound>0,'sentiment_type']='POSITIVE'
df.loc[df.compound==0,'sentiment_type']='NEUTRAL'
df.loc[df.compound<0,'sentiment_type']='NEGATIVE'
df.sentiment_type.value_counts(normalize=True).plot(kind='bar',title="Sentiment Analysis of IRA Tweets")

df_liar['scores'] = df_liar['text'].apply(lambda text: sid.polarity_scores(text))
df = df_liar

df['compound'] = df['scores'].apply(lambda score_dict: score_dict['compound'])
df['sentiment_type']=''
df['compound']
df.loc[df.compound>0,'sentiment_type']='POSITIVE'
df.loc[df.compound==0,'sentiment_type']='NEUTRAL'
df.loc[df.compound<0,'sentiment_type']='NEGATIVE'
df.sentiment_type.value_counts(normalize=True).plot(kind='bar',title="Sentiment Analysis of LIAR Text")

df = df.loc[df['ratingnum'] == 5]
df
df.loc[df.compound>0,'sentiment_type']='POSITIVE'
df.loc[df.compound==0,'sentiment_type']='NEUTRAL'
df.loc[df.compound<0,'sentiment_type']='NEGATIVE'
df.sentiment_type.value_counts(normalize=True).plot(kind='bar',title="Sentiment Analysis of True LIAR Text")

df['ratingnum'].value_counts()
import tweepy

consumer_key = 'cJSMBI4xpLjIEcW3RN8qlh9pA'
consumer_secret = '<KEY>'
access_key= '<KEY>'
access_secret = '<KEY>'

bearer_token = '<PASSWORD>AAAAAGsEQQEAAAAAYR2NNUx%2BKkBujAt7Ng1MpbIrWvQ%<KEY>'
# authorization of consumer key and consumer secret
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
  
# set access to user's access key and access secret 
auth.set_access_token(access_key, access_secret)
  
# calling the api 
api = tweepy.API(auth)
# We import our access keys:


# API's setup:
def twitter_setup():
    """
    Utility function to setup the Twitter's API
    with our access keys provided.
    """
    # Authentication and access using keys:
    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_key, access_secret)

    # Return API with authentication:
    api = tweepy.API(auth)
    return api
# We create an extractor object:
extractor = twitter_setup()

# Only iterate through the first 3 pages
for page in tweepy.Cursor(api.user_timeline).pages(3):
    process_page(page)
# We create a tweet list as follows:
tweets = extractor.user_timeline(screen_name="AP", count=3000)
print("Number of tweets extracted: {}.\n".format(len(tweets)))

# We print the most recent 5 tweets:
print("5 recent tweets:\n")
for tweet in tweets[:5]:
    print(tweet.text)
    print()
# We create a pandas dataframe as follows:
data = pd.DataFrame(data=[tweet.text for tweet in tweets], columns=['Tweets'])

# We display the first 10 elements of the dataframe:
display(data.head(10))
# We add relevant data:
data['len']  = np.array([len(tweet.text) for tweet in tweets])
data['ID']   = np.array([tweet.id for tweet in tweets])
data['Date'] = np.array([tweet.created_at for tweet in tweets])
data['Source'] = np.array([tweet.source for tweet in tweets])
data['Likes']  = np.array([tweet.favorite_count for tweet in tweets])
data['RTs']    = np.array([tweet.retweet_count for tweet in tweets])
# Display of first 10 elements from dataframe:
display(data.head(10))
df_AP = data
tweets = extractor.user_timeline(screen_name="Reuters", count=3000)
print("Number of tweets extracted: {}.\n".format(len(tweets)))

# We print the most recent 5 tweets:
print("5 recent tweets:\n")
for tweet in tweets[:5]:
    print(tweet.text)
    print()
# We create a pandas dataframe as follows:
data = pd.DataFrame(data=[tweet.text for tweet in tweets], columns=['Tweets'])

# We display the first 10 elements of the dataframe:
display(data.head(10))
# We add relevant data:
data['len']  = np.array([len(tweet.text) for tweet in tweets])
data['ID']   = np.array([tweet.id for tweet in tweets])
data['Date'] = np.array([tweet.created_at for tweet in tweets])
data['Source'] = np.array([tweet.source for tweet in tweets])
data['Likes']  = np.array([tweet.favorite_count for tweet in tweets])
data['RTs']    = np.array([tweet.retweet_count for tweet in tweets])
# Display of first 10 elements from dataframe:
display(data.head(10))
df_Rueters = data
df_Rueters.head()

tweets = extractor.user_timeline(screen_name="USATODAY", count=3000)
print("Number of tweets extracted: {}.\n".format(len(tweets)))

# We print the most recent 5 tweets:


# We create a pandas dataframe as follows:
data = pd.DataFrame(data=[tweet.text for tweet in tweets], columns=['Tweets'])

# We display the first 10 elements of the dataframe:

# We add relevant data:
data['len']  = np.array([len(tweet.text) for tweet in tweets])
data['ID']   = np.array([tweet.id for tweet in tweets])
data['Date'] = np.array([tweet.created_at for tweet in tweets])
data['Source'] = np.array([tweet.source for tweet in tweets])
data['Likes']  = np.array([tweet.favorite_count for tweet in tweets])
data['RTs']    = np.array([tweet.retweet_count for tweet in tweets])

df_USATODAY = data

df_USATODAY.head()
tweets = extractor.user_timeline(screen_name="WSJ", count=3000)
print("Number of tweets extracted: {}.\n".format(len(tweets)))

# We print the most recent 5 tweets:


# We create a pandas dataframe as follows:
data = pd.DataFrame(data=[tweet.text for tweet in tweets], columns=['Tweets'])

# We display the first 10 elements of the dataframe:

# We add relevant data:
data['len']  = np.array([len(tweet.text) for tweet in tweets])
data['ID']   = np.array([tweet.id for tweet in tweets])
data['Date'] = np.array([tweet.created_at for tweet in tweets])
data['Source'] = np.array([tweet.source for tweet in tweets])
data['Likes']  = np.array([tweet.favorite_count for tweet in tweets])
data['RTs']    = np.array([tweet.retweet_count for tweet in tweets])

df_WSJ = data

df_WSJ.head()
tweets = extractor.user_timeline(screen_name="BBCNews", count=3000)
print("Number of tweets extracted: {}.\n".format(len(tweets)))

# We print the most recent 5 tweets:


# We create a pandas dataframe as follows:
data = pd.DataFrame(data=[tweet.text for tweet in tweets], columns=['Tweets'])

# We display the first 10 elements of the dataframe:

# We add relevant data:
data['len']  = np.array([len(tweet.text) for tweet in tweets])
data['ID']   = np.array([tweet.id for tweet in tweets])
data['Date'] = np.array([tweet.created_at for tweet in tweets])
data['Source'] = np.array([tweet.source for tweet in tweets])
data['Likes']  = np.array([tweet.favorite_count for tweet in tweets])
data['RTs']    = np.array([tweet.retweet_count for tweet in tweets])

df_BBC = data

df_BBC.head()
dfaprut = df_AP.append(df_Rueters)    #append all the datasets into a single dataframe
dfaru = dfaprut.append(df_USATODAY)
dfaruw = dfaru.append(df_WSJ)
df_truth = dfaruw.append(df_BBC)
df_truth.head()
df_truth.info()
!pip install searchtweets
from searchtweets import load_credentials
import os
os.environ["SEARCHTWEETS_USERNAME"] = "<ENV_USERNAME>"
os.environ["SEARCHTWEETS_PASSWORD"] = "<ENV_PW>"
os.environ["SEARCHTWEETS_ENDPOINT"] = "<https://api.twitter.com/1.1/tweets/search/fullarchive/searchTweets2.json>"

load_credentials(filename="nothing_here.yaml", yaml_key="no_key_here")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13701721010485335, 'fraction_numerical': 0.02097066761049418, 'mean_word_length': 5.7920673076923075, 'pattern_counts': {'":': 3, '<': 32, '<?xml version=': 0, '>': 30, 'https://': 19, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '208749', 'license': 'Apache-2.0', 'n_tokens_mistral': 8320, 'n_tokens_neox': 7834, 'n_words': 1863, 'path': 'InfoWarfareRussiaNLPv2.ipynb', 'repo_name': 'twiese86/Cirq', 'size': '22846'}