<jupyter_start><jupyter_text>Set notebook options<jupyter_code>pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 15)
pd.set_option('display.precision', 3)
pd.set_option('max_colwidth', 1000)<jupyter_output><empty_output><jupyter_text>Scrape data if not already downloaded<jupyter_code>names = '<NAME> <NAME> <NAME>'.split()
filenames = ['data/' + name + '.csv' for name in names]

for idx, file in enumerate(filenames):
    if not os.path.exists('data/kim.csv'):
        print('Scraping Kim... (write scrape code)')
    else:
        name = names[idx]
        print(f'{name} is a basic bitch (already downloaded)')<jupyter_output>kim is a basic bitch (already downloaded)
khloe is a basic bitch (already downloaded)
kourtney is a basic bitch (already downloaded)
kris is a basic bitch (already downloaded)
kylie is a basic bitch (already downloaded)
kendall is a basic bitch (already downloaded)
cailtyn is a basic bitch (already downloaded)
rob is a basic bitch (already downloaded)<jupyter_text>Read in and combine data<jupyter_code>#caitlyn  = pd.read_csv('data/caitlyn.csv' , low_memory=False)
kendall  = pd.read_csv('data/kendall.csv' , low_memory=False)
kylie    = pd.read_csv('data/kylie.csv'   , low_memory=False)
kim      = pd.read_csv('data/kim.csv'     , low_memory=False)
kourtney = pd.read_csv('data/kourtney.csv', low_memory=False)
khloe    = pd.read_csv('data/khloe.csv'   , low_memory=False)
kris     = pd.read_csv('data/kris.csv'    , low_memory=False)
#rob      = pd.read_csv('data/rob.csv'     , low_memory=False)
#kardashians_ = pd.concat([caitlyn, kendall, kylie, kim, kourtney, khloe, kris, rob])
kardashians_ = pd.concat([kendall, kylie, kim, kourtney, khloe, kris])
kardashians = kardashians_[['date', 'time', 'username', 'tweet']]
raw_tweets = copy.deepcopy(kardashians['tweet'])
raw_tweets.head(5)
# Atwood
# Trump
raw_tweets = raw_tweets.append(pd.Series(["<NAME>'s memoir about @ViragoBooks, forthcoming from @OUP, is now called A Bite of the Apple. All an apple should be: crisp, tart but sweet, steeped in mysterious history and tangled symbolism, and not a bad missile when it comes to alleyway combat. Oh, and delicious!",
                             "The Fake News LameStream Media is doing everything possible the “create” a U.S. recession, even though the numbers & facts are working totally in the opposite direction. They would be willing to hurt many people, but that doesn’t matter to them. Our Economy is sooo strong, sorry!"]))
raw_tweets = raw_tweets[0]
raw_tweets.tail()
raw_tweets = raw_tweets.reset_index()
tweets = copy.deepcopy(raw_tweets)

tweets = tweets.str.replace('[a-z]*\d+[a-z]*', '')
tweets = tweets.str.replace(r'http\S+', '')
tweets = tweets.str.replace(r'pic.twitter.*', '')
tweets = tweets.str.replace('(@[A-Za-z0-9]+)', '')
tweets = tweets.str.replace('(#[A-Za-z0-9]+)', '')
tweets = tweets.str.replace(r'[^\w\s]', '')
tweets = tweets.str.replace('\d+', '')
tweets = tweets.str.replace('(\s)_\w+', '')

tweets.head(5)<jupyter_output><empty_output><jupyter_text>Try topic modeling<jupyter_code># gensim
#from gensim import corpora, models, similarities, matutils

# sklearn
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer


# logging for gensim (set to INFO)
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)


# USE TFIDF and LDA
# tweet | username | topic1 | topic2 | ... 
# Grab tweets (data) and corresponding usernames (targets) seperately, but PRESERVE ROW ORDER
#tweets = kardashians['tweet']
usernames = kardashians['username']
# Create a CountVectorizer for parsing/counting words
count_vectorizer = TfidfVectorizer(
                                   tokenizer=None,
                                   ngram_range=(1,3),
                                   stop_words=None,
                                   min_df=10,
                                   #strip_accents='unicode',
                                   #sublinear_tf=True
    
)

# count_vectorizer = CountVectorizer(
#                                    analyzer='word',
#                                    ngram_range=(1, 2),  
#                                    stop_words='english',
#                                    token_pattern="\\b[a-z][a-z]+\\b"
#                                   )

count_vectorizer.fit(tweets)
# Split data into train set and test set

# KARDASHIAN BY TIME (RECENT / TOTAL / ETC...)
# Create the transposed 
doc_term = count_vectorizer.transform(tweets) # JUST TRANSFORM TWEETS FROM OTHERS
doc_term.shape
# Convert sparse matrix of counts to a gensim corpus
corpus = matutils.Sparse2Corpus(doc_term)
id2word = dict((v, k) for k, v in count_vectorizer.vocabulary_.items())
len(id2word)
# Create LDA model (equivalent to "fit" in sklearn)

lda = LatentDirichletAllocation(n_components=20, max_iter=6, learning_method='online', learning_offset=50.,random_state=0).fit(doc_term)

#lda = models.LdaModel(corpus=corpus, num_topics=10, id2word=id2word, passes=5)
len(lda.components_[0])
count_vectorizer.get_feature_names()[:6]
n_top_words = 10
tfidf_feature_names = count_vectorizer.get_feature_names()

def display_topics(model, feature_names, no_top_words, topic_names=None):
    for ix, topic in enumerate(model.components_):
        if not topic_names or not topic_names[ix]:
            print("\nTopic ", ix+1)
        else:
            print("\nTopic: '", topic_names[ix], "'")
        print(", ".join([feature_names[i] for i in topic.argsort()[:-no_top_words - 1:-1]]))

display_topics(lda, tfidf_feature_names, n_top_words)
# Transform the docs from the word space to the topic space (like "transform" in sklearn)
lda_corpus = lda.transform(doc_term)
lda_corpus
# Store the documents' topic vectors in a list so we can take a peak
lda_docs = [doc for doc in lda_corpus]
# Check out the document vectors in the topic space for the first 5 documents
lda_docs[0]
 topic_names = ['topic01', 'topic02', 'topic03', 'topic04', 'topic05', 'topic06', 'topic07', 'topic08',
                'topic09', 'topic10', 'topic11' ,'topic12' ,'topic13' ,'topic14' ,'topic15' ,'topic16',
                'topic17', 'topic18', 'topic19', 'topic20']

# topic_names = ['topic00', 'topic01', 'topic02', 'topic03', 'topic04', 'topic05', 'topic06', 'topic07',
#                'topic08', 'topic09']

# topic_names = ['topic00', 'topic01', 'topic02', 'topic03', 'topic04', 'topic05', 'topic06', 'topic07',
#                'topic08', 'topic09', 'topic10', 'topic11', 'topic12', 'topic13', 'topic14']

topics_df = pd.DataFrame(lda_docs, columns=topic_names)
topics_df.shape
atwood_trump = topics_df.iloc[-2:,:]
topics_df = topics_df.iloc[:-2,:]
kardashians.shape
topics_df.shape
kardashians.reset_index(inplace=True, drop=True)
topics_df.reset_index(inplace=True, drop=True)
lda_kardash = pd.concat([kardashians, topics_df], axis=1)
lda_kardash.head(2)
lda_kardash.shape
lda_kardash.groupby('username').mean()<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code># Train-test split
lda_kardash.username[lda_kardash.username == 'krisjenner'] = 1
lda_kardash.username[lda_kardash.username == 'kyliejenner'] = 2
lda_kardash.username[lda_kardash.username == 'kendalljenner'] = 3
lda_kardash.username[lda_kardash.username == 'kimkardashian'] = 4
lda_kardash.username[lda_kardash.username == 'khloekardashian'] = 5
lda_kardash.username[lda_kardash.username == 'kourtneykardash'] = 6
y = lda_kardash.username
X = lda_kardash.iloc[:, 4:]
lda_kardash.shape
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
y_train = y_train.astype('int')
y_test  = y_test.astype('int')
lr_model = LogisticRegression(solver='lbfgs', multi_class='multinomial')
y_train = y_train.astype('int')
y_test  = y_test.astype('int')
lr_fit = lr_model.fit(X_train, y_train)
y_predict = lr_model.predict(X_test)
y_predict
lr_model.score(X_test, y_test)
target_names = ['krisjenner', 'kyliejenner', 'kendalljenner', 'kimkardashian',
                'khloekardashian', 'kourtneykardash']

print(classification_report(y_test, y_predict, target_names=target_names))<jupyter_output>precision    recall  f1-score   support

     krisjenner       0.00      0.00      0.00      1822
    kyliejenner       0.00      0.00      0.00      2053
  kendalljenner       0.00      0.00      0.00      1594
  kimkardashian       0.50      0.00      0.00      5399
khloekardashian       0.45      1.00      0.62     10999
kourtneykardash       0.00      0.00      0.00      2412

      micro avg       0.45      0.45      0.45     24279
      macro avg       0.16      0.17      0.10     24279
   weighted avg       0.32      0.45      0.28     24279<jupyter_text>Random Forest<jupyter_code># Create the model with 100 trees
rfm = RandomForestClassifier(
    n_estimators=500,
    #criterion='gini',
    #max_depth=None,
    #min_samples_split=2,
    #min_samples_leaf=1,
    #min_weight_fraction_leaf=0,
    max_features='sqrt',
    #max_leaf_nodes=None,
    #min_impurity_decrease=0,
    #min_impurity_split=1e-7,
    #bootstrap=True,
    #oob_score=False,
    #n_jobs=None,
    #random_state=None,
    #verbose=0,
    #warm_start=False,
    class_weight='balanced'
)

# Fit on training data
rfm.fit(X_train, y_train)
X_test
X_test.concat(atwood_trump)
X_test.tail()
# train set
train_rfm_predictions = rfm.predict(X_train)
train_rfm_probs = rfm.predict_proba(X_train)[:, 1]

# test set
y_predict = rfm.predict(X_test)
rfm_probs = rfm.predict_proba(X_test)[:, 1]
at_tr_predict = rfm.predict(atwood_trump)
at_tr_predict
target_names = ['krisjenner', 'kyliejenner', 'kendalljenner', 'kimkardashian',
                'khloekardashian', 'kourtneykardash']

print(classification_report(y_test, y_predict, target_names=target_names))<jupyter_output>precision    recall  f1-score   support

     krisjenner       0.27      0.11      0.15      1822
    kyliejenner       0.21      0.21      0.21      2053
  kendalljenner       0.21      0.11      0.15      1594
  kimkardashian       0.35      0.14      0.20      5399
khloekardashian       0.50      0.81      0.62     10999
kourtneykardash       0.21      0.06      0.10      2412

      micro avg       0.44      0.44      0.44     24279
      macro avg       0.29      0.24      0.24     24279
   weighted avg       0.38      0.44      0.37     24279<jupyter_text>Visualize<jupyter_code>from sklearn import manifold
from sklearn.neighbors import NearestNeighbors
from sklearn.manifold import TSNE

import numpy as np
import bokeh.plotting as bp
from bokeh.plotting import save
from bokeh.models import HoverTool
topic_vectors = np.array(topics_df)
topic_vectors
tsne_model = TSNE(n_components=2, verbose=1, random_state=0, angle=.99, init='pca')
tsne_lda = tsne_model.fit_transform(topic_vectors)
n_top_words = 5

colormap = np.array([
    "#1f77b4", "#aec7e8", "#ff7f0e", "#ffbb78", "#2ca02c",
    "#98df8a", "#d62728", "#ff9896", "#9467bd", "#c5b0d5",
    "#8c564b", "#c49c94", "#e377c2", "#f7b6d2", "#7f7f7f",
    "#c7c7c7", "#bcbd22", "#dbdb8d", "#17becf", "#9edae5"
])
_lda_keys = []
for i in xrange(topic_vectors.shape[0]):
  _lda_keys +=  _topics[i].argmax(),
nn = NearestNeighbors(metric='cosine', algorithm='brute')
nn.fit(topic_vectors)

word_indices = set()

key_words = ['sweet', 'proud', 'baby', 'khloe', 'thanks', 'yes', 'today', 'miss', 'good',
             'know', 'thank', 'fun', 'dolls', 'amazing', 'love', 'hi', 'muah', 'true', 'wow', 'yay']
#key_words = ["monster", "margaret", "elizabeth", "victor", "robert", "frankenstein", "henry", "justine", "beaufort", "caroline", "william"]
for key in key_words:
    word_index = count_vectorizer.vocabulary_[key]
    dist, index = nn.kneighbors(topic_vectors[[word_index], :], n_neighbors=10)
    word_indices.add(word_index)
    word_indices.update(list(index.flat))
    
word_indices = list(word_indices)


target_vecs = topic_vectors[word_indices,:]
#figsize(10,10)

vecs_emb = manifold.TSNE( metric="cosine", n_iter=3000).fit_transform(target_vecs)

sns.scatterplot(x=vecs_emb[:,0], y=vecs_emb[:,1],marker='.' )
for i, row in enumerate(vecs_emb):
    word_i = word_indices[i]
    word = count_vectorizer.get_feature_names()[word_i]
    if word in key_words:
        annotate(word, row, color="red", bbox=dict(boxstyle="round", fc="w", ec="k", alpha=.5), zorder=10)
    else:
        annotate(word, row, bbox=dict(boxstyle="round", fc="w", ec="k", alpha=.5), zorder=-1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13128317202622541, 'fraction_numerical': 0.03847955042147986, 'mean_word_length': 3.782754759238522, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '345062', 'license': 'MIT', 'n_tokens_mistral': 5005, 'n_tokens_neox': 4674, 'n_words': 985, 'path': 'get_data.ipynb', 'repo_name': 'jlehnersd/kardashians', 'size': '12850'}