<jupyter_start><jupyter_text>Text Minning<jupyter_code>#!pip insall tweepy - install tweepy with pip
#conda install -c conda-forge tweepy --> install tweepy for anaconda
# conda install -c conda-forge/label/cf201901 tweepy --> install tweepy for anaconda
import tweepy, codecs


consumer_key = ''
consumer_secret = ''
access_token = ''
access_token_secret = ''


auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
tweets = api.search(q = "#datascience", lang = "en", count = 5000)
import pandas as pd
def hashtag_df(tweets):
    id_list = [tweet.id for tweet  in tweets]
    df = pd.DataFrame(id_list, columns = ["id"])
    
    df["text"] = [tweet.text for tweet in tweets]
    df["created_at"] = [tweet.created_at for tweet in tweets]
    df["retweeted"] = [tweet.retweeted for tweet in tweets]
    df["retweet_count"] = [tweet.retweet_count for tweet in tweets]
    df["source"] = [tweet.source for tweet in tweets]
    df["user_screen_name"] = [tweet.author.screen_name for tweet in tweets]
    df["user_followers_count"] = [tweet.author.followers_count for tweet in tweets]
    df["user_location"] = [tweet.author.location for tweet in tweets]
    df["Hashtags"] = [tweet.entities.get('hashtags') for tweet in tweets]
    
    return df
df = hashtag_df(tweets)
df = hashtag_df(tweets)

#convert all letters to the same size (large, small transformation)
df['text'] = df['text'].apply(lambda x: " ".join(x.lower() for x in x.split()))

#remove punctuation
df['text'] = df['text'].str.replace('[^\w\s]','')

#sremoving numbers
df['text'] = df['text'].str.replace('\d','')

#stopwords
#conda install -c anaconda nltk  --> for anaconda
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
sw = stopwords.words('english')
df['text'] = df['text'].apply(lambda x: " ".join(x for x in x.split() if x not in sw))

#lemmi
#conda install -c conda-forge textblob  --> for anaconda
from textblob import Word
nltk.download('wordnet')
df['text'] = df['text'].apply(lambda x: " ".join([Word(word).lemmatize() for word in x.split()])) 

#npunctuations
df['text'] = df['text'].str.replace('rt','')
df["text"]
freq_df = df["text"].apply(lambda x: pd.value_counts(x.split(" "))).sum(axis =0).reset_index()
freq_df.columns = ["words","frequencies"]
freq_df.shape
freq_df[freq_df.frequencies > freq_df.frequencies.mean() + 
        freq_df.frequencies.std()]
keywords = freq_df[freq_df.frequencies > freq_df.frequencies.mean() + 
        freq_df.frequencies.std()]
keywords.plot.bar(x = "words",y = "frequencies");<jupyter_output><empty_output><jupyter_text>Word Cloud<jupyter_code>import numpy as np
import pandas as pd
from os import path
from PIL import Image
#conda install -c conda-forge wordcloud --> for anaconda
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import matplotlib.pyplot as plt
text = " ".join(i for i in df.text)
text
wordcloud = WordCloud(background_color = "white").generate(text)
plt.imshow(wordcloud, interpolation = "bilinear")
plt.axis("off")
plt.tight_layout(pad = 0)
plt.show()
image_mask = np.array(Image.open("network.jpg"))
wc = WordCloud(background_color = "white", 
               max_words = 1000, 
               mask = image_mask,
              contour_width = 3)

wc.generate(text)
plt.figure(figsize=(10,10))
plt.imshow(wc, interpolation = "bilinear")
plt.axis("off")
plt.tight_layout(pad = 0)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14938236139040506, 'fraction_numerical': 0.006320022981901752, 'mean_word_length': 4.813021702838063, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '384461', 'license': 'Apache-2.0', 'n_tokens_mistral': 1300, 'n_tokens_neox': 1228, 'n_words': 293, 'path': 'text minning for data science/text manning  and wold cloud.ipynb', 'repo_name': 'ahmterdgn903/data_science', 'size': '3481'}