<jupyter_start><jupyter_text>Install twint, optimus and reqs<jupyter_code>!pip install --user -r requirements.txt
!pip install --upgrade --user -e git+https://github.com/twintproject/twint.git@origin/master#egg=twint
!mv src/twint .
!rm -r src
%load_ext autoreload
%autoreload 2

import sys
sys.path.append("twint/")
import twint
from optimus import Optimus
op = Optimus()<jupyter_output><empty_output><jupyter_text>Setup twint<jupyter_code># Set up TWINT config
c = twint.Config()
# Solve compatibility issues with notebooks and RunTime errors.
import nest_asyncio
nest_asyncio.apply()<jupyter_output><empty_output><jupyter_text>Search for data science tweets<jupyter_code>c.Search = "data science"
# Custom output format
c.Format = "Username: {username} |  Tweet: {tweet}"
c.Limit = 1
c.Pandas = True

twint.run.Search(c)<jupyter_output>Username: InfoProNetwork |  Tweet: IAM Platform Curated Retweet:  Via:  https://twitter.com/armaninspace  #ArtificialIntelligence (AI): What About The User Experience?  https://www.forbes.com/sites/tomtaulli/2019/04/27/artificial-intelligence-ai-what-about-the-user-experience/ … #AI #DataScience #MachineLearning #BigData #DeepLearning #Robots #IoT #ML #DL #IAMPlatform #TopInfluence #ArtificialIntelligence
Username: thisismetis |  Tweet: Seattle Data Science Career Advice: Landing a Job in The Emerald City. (Tips from Metis Seattle Career Advisor Marybeth Redmond) –  https://bit.ly/2IYjzaj  pic.twitter.com/98hMYZVxsu
Username: thisismetis |  Tweet: This webinar/workshop is designed for business leaders, data science managers, and decision makers who want to build effective AI and data science capabilities for their organization. Register here:  https://bit.ly/2GDQeQT  pic.twitter.com/xENQ0Dtv1X
Username: ihutagalung |  Tweet: Contoh yang menarik dari "sport science", kali ini dari sisi [...]<jupyter_text>Saving results into Pandas<jupyter_code>def available_columns():
    return twint.output.panda.Tweets_df.columns
def twint_to_pandas(columns):
    return twint.output.panda.Tweets_df[columns]
available_columns()
df_pd = twint_to_pandas(["date", "username", "tweet", "hashtags", "nlikes"])
df_pd<jupyter_output><empty_output><jupyter_text>Sentiment analysis  Save data to Optimus<jupyter_code>df = op.create.data_frame(pdf= df_pd)<jupyter_output><empty_output><jupyter_text>Clean tweets<jupyter_code>clean_tweets = df.cols.remove_accents("tweet") \
                 .cols.remove_special_chars("tweet")
clean_tweets.count()
tweets = clean_tweets.select("tweet").rdd.flatMap(lambda x: x).collect()
tweets<jupyter_output><empty_output><jupyter_text>Analyze sentiment of tweets<jupyter_code>from textblob import TextBlob
from IPython.display import Markdown, display
def printmd(string, color=None):
    colorstr = "<span style='color:{}'>{}</span>".format(color, string)
    display(Markdown(colorstr))
for tweet in tweets:
    print(tweet)
    analysis = TextBlob(tweet)
    print(analysis.sentiment)
    if analysis.sentiment[0]>0:
        printmd('Positive', color="green")
    elif analysis.sentiment[0]<0:
        printmd('Negative', color="red")
    else:
        printmd("Neutral", color="grey")
        print("")<jupyter_output>IAM Platform Curated Retweet  Via  httpstwittercomarmaninspace  ArtificialIntelligence AI What About The User Experience  httpswwwforbescomsitestomtaulli20190427artificialintelligenceaiwhatabouttheuserexperience  AI DataScience MachineLearning BigData DeepLearning Robots IoT ML DL IAMPlatform TopInfluence ArtificialIntelligence
Sentiment(polarity=0.0, subjectivity=0.0)<jupyter_text>Add sentiments to dataframe<jupyter_code>from pyspark.sql.functions import udf
def apply_blob(sentence):
    temp = TextBlob(sentence).sentiment[0]
    if temp == 0.0:
        return 0.0 # Neutral
    elif temp >= 0.0:
        return 1.0 # Positive
    else:
        return 2.0 # Negative
sentiment = udf(apply_blob)
clean_tweets.withColumn("sentiment", sentiment(clean_tweets['tweet'])).show()<jupyter_output>+-------------------+---------------+--------------------+--------------------+------+---------+
|               date|       username|               tweet|            hashtags|nlikes|sentiment|
+-------------------+---------------+--------------------+--------------------+------+---------+
|2019-05-04 16:00:19| InfoProNetwork|IAM Platform Cura...|[#artificialintel...|     0|      0.0|
|2019-05-04 16:00:07|    thisismetis|Seattle Data Scie...|                  []|     0|      0.0|
|2019-05-04 16:00:02|    thisismetis|This webinarworks...|                  []|     0|      1.0|
|2019-05-04 16:00:01|    ihutagalung|Contoh yang menar...|                  []|     0|      0.0|
|2019-05-04 15:59:56|        byLilyV|Complete handson ...|[#python, #machin...|     0|      2.0|
|2019-05-04 15:59:35| aSweetEspresso|We share criminal...|                  []|     0|      1.0|
|2019-05-04 15:59:06|  DataPanoramic|Value of Manufact...|[#bigdata, #datas...|     0|      0.0|
|2019-05-04 15:58:58|     Mall[...]<jupyter_text>Making the code modular  Setup everything<jupyter_code>%load_ext autoreload
%autoreload 2

# Import twint
import sys
sys.path.append("twint/")

# Set up TWINT config
import twint
c = twint.Config()

# Other imports
import seaborn as sns
import os
from optimus import Optimus
op = Optimus()

# Solve compatibility issues with notebooks and RunTime errors.
import nest_asyncio
nest_asyncio.apply()

# Disable annoying printing

class HiddenPrints:
    def __enter__(self):
        self._original_stdout = sys.stdout
        sys.stdout = open(os.devnull, 'w')

    def __exit__(self, exc_type, exc_val, exc_tb):
        sys.stdout.close()
        sys.stdout = self._original_stdout<jupyter_output>/opt/conda/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!
  RequestsDependencyWarning)<jupyter_text>The code with the magic<jupyter_code>from textblob import TextBlob
from pyspark.sql.functions import udf
from pyspark.sql.types import DoubleType

# Function to get sentiment 
def apply_blob(sentence):
    temp = TextBlob(sentence).sentiment[0]
    if temp == 0.0:
        return 0.0 # Neutral
    elif temp >= 0.0:
        return 1.0 # Positive
    else:
        return 2.0 # Negative

# UDF to write sentiment on DF
sentiment = udf(apply_blob, DoubleType())

# Transform result to pandas
def twint_to_pandas(columns):
    return twint.output.panda.Tweets_df[columns]

def tweets_sentiment(search, limit=1):
    c.Search = search
    # Custom output format
    c.Format = "Username: {username} |  Tweet: {tweet}"
    c.Limit = limit
    c.Pandas = True
    with HiddenPrints():
        print(twint.run.Search(c))
    
    # Transform tweets to pandas DF
    df_pd = twint_to_pandas(["date", "username", "tweet", "hashtags", "nlikes"])
    
    # Transform Pandas DF to Optimus/Spark DF
    df = op.create.data_frame(pdf= df_pd)
    
    # Clean tweets
    clean_tweets = df.cols.remove_accents("tweet") \
                 .cols.remove_special_chars("tweet")
    
    # Add sentiment to final DF
    return clean_tweets.withColumn("sentiment", sentiment(clean_tweets['tweet']))
df_result = tweets_sentiment("data science", limit=1)
df_result.show()
df_result.count()
df_result.printSchema()<jupyter_output>root
 |-- date: string (nullable = true)
 |-- username: string (nullable = true)
 |-- tweet: string (nullable = true)
 |-- hashtags: array (nullable = true)
 |    |-- element: string (containsNull = true)
 |-- nlikes: long (nullable = true)
 |-- sentiment: double (nullable = true)<jupyter_text>Lets see the distribution of the sentiments<jupyter_code>df_res_pandas = df_result.toPandas()
sns.distplot(df_res_pandas['sentiment'])
sns.set(rc={'figure.figsize':(11.7,8.27)})<jupyter_output><empty_output><jupyter_text>Lets do more with Twint<jupyter_code>%load_ext autoreload
%autoreload 2

import sys, os
sys.path.append("twint/")

import twint
import numpy as np
from PIL import Image
from os import path

# Solve compatibility issues with notebooks and RunTime errors.
import nest_asyncio
nest_asyncio.apply()
# Disable annoying printing

class HiddenPrints:
    def __enter__(self):
        self._original_stdout = sys.stdout
        sys.stdout = open(os.devnull, 'w')

    def __exit__(self, exc_type, exc_val, exc_tb):
        sys.stdout.close()
        sys.stdout = self._original_stdout<jupyter_output><empty_output><jupyter_text>Get tweets easily<jupyter_code>def get_tweets(search, limit=100):
    c = twint.Config()
    c.Search = search
    c.Limit = limit
    c.Pandas = True
    c.Pandas_clean = True

    with HiddenPrints():
        print(twint.run.Search(c))
    return twint.output.panda.Tweets_df[["username","tweet"]]
tweets = get_tweets("data science", limit=10000)
tweets.count()<jupyter_output><empty_output><jupyter_text>Generate word cloud<jupyter_code>from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import matplotlib.pyplot as plt
%matplotlib inline

text = tweets.tweet.values

# adding movie script specific stopwords
stopwords = set(STOPWORDS)
stopwords.add("https")
stopwords.add("xa0")
stopwords.add("xa0'")
stopwords.add("bitly")
stopwords.add("bit")
stopwords.add("ly")
stopwords.add("twitter")
stopwords.add("pic")

wordcloud = WordCloud(
    background_color = 'black',
    width = 1000,
    height = 500,
    stopwords = stopwords).generate(str(text))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.rcParams['figure.figsize'] = [10, 10]
def generate_word_cloud(tweets):
    
    # Getting the text out of the tweets
    text = tweets.tweet.values
    
    # adding movie script specific stopwords
    stopwords = set(STOPWORDS)
    stopwords.add("https")
    stopwords.add("xa0")
    stopwords.add("xa0'")
    stopwords.add("bitly")
    stopwords.add("bit")
    stopwords.add("ly")
    stopwords.add("twitter")
    stopwords.add("pic")

    wordcloud = WordCloud(
        background_color = 'black',
        width = 1000,
        height = 500,
        stopwords = stopwords).generate(str(text))
    
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis("off")
    plt.rcParams['figure.figsize'] = [10, 10]<jupyter_output><empty_output><jupyter_text>Other example<jupyter_code>tweets = get_tweets("artificial intelligence", limit=1000)
generate_word_cloud(tweets)<jupyter_output><empty_output><jupyter_text>Scaling up<jupyter_code>df_result = tweets_sentiment("data science", limit=10000)
df_result.show()
df_result.count()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13543247344461304, 'fraction_numerical': 0.023899848254931716, 'mean_word_length': 3.929873772791024, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 65, 'https://': 5, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '331604', 'license': 'Apache-2.0', 'n_tokens_mistral': 3790, 'n_tokens_neox': 3539, 'n_words': 771, 'path': 'AnalyzeTweets.ipynb', 'repo_name': 'eugeniousone/twitter_optimus_twint', 'size': '10544'}