<jupyter_start><jupyter_text>Social Web -  TwitterTwitter implements OAuth 1.0A as its standard authentication mechanism, and in order to use it to make requests to Twitter's API, you'll need to go to https://dev.twitter.com/apps and create a sample application. Twitter examples from the python-twitter API  [https://github.com/ideoforms/python-twitter-examples](https://github.com/ideoforms/python-twitter-examples)    Authorizing an application to access Twitter account data<jupyter_code>import twitter # pip install twitter

# Go to http://dev.twitter.com/apps/new to create an app and get values
# for these credentials, which you'll need to provide in place of these
# empty string values that are defined as placeholders.
# See https://dev.twitter.com/docs/auth/oauth for more information 
# on Twitter's OAuth implementation.

CONSUMER_KEY = '<KEY>'
CONSUMER_SECRET = '<KEY>'
OAUTH_TOKEN = '34749<PASSWORD>-<KEY>'
OAUTH_TOKEN_SECRET = '<KEY>'

auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET,
                           CONSUMER_KEY, CONSUMER_SECRET)

twitter_api = twitter.Twitter(auth=auth)

# Nothing to see by displaying twitter_api except that it's now a
# defined variable

print(twitter_api)<jupyter_output><twitter.api.Twitter object at 0x102cf5320><jupyter_text>Retrieving trends<jupyter_code>#-----------------------------------------------------------------------
# retrieve global trends.
# other localised trends can be specified by looking up WOE IDs:
#   http://developer.yahoo.com/geo/geoplanet/
# twitter API docs: https://dev.twitter.com/rest/reference/get/trends/place
#-----------------------------------------------------------------------
def twitter_trends(place, woe_id):
    results = twitter_api.trends.place(_id = woe_id)
    print ("%s Trends"%place)
    t=[]
    for location in results:
        for trend in location["trends"]:
            print (" - %s" % trend["name"])
            t.append(trend["name"])
    return t
woe_ids={'UK':23424975,'US':23424977,'WORLD':1}
uk_trends=twitter_trends('UK',woe_ids['UK'])
us_trends=twitter_trends('US',woe_ids['US'])
world_trends=twitter_trends('WORLD',woe_ids['WORLD'])
def intersect(a, b):
    return list(set(a) & set(b))
common_trends = intersect(us_trends,uk_trends)
print(common_trends)<jupyter_output>['#WorldSeries', '#WhatImGoodAtIn5Words']<jupyter_text>Getting Tweets<jupyter_code>import json

q = '#DeepLearning' 

n = 55

from urllib.parse import unquote

# See https://dev.twitter.com/rest/reference/get/search/tweets

search_results = twitter_api.search.tweets(q=q, count=n)

statuses = search_results['statuses']

for _ in range(5):
    try:
        next_results = search_results['search_metadata']['next_results']
    except KeyError as e: # No more results when next_results doesn't exist
        break
        
    # Create a dictionary from next_results, which has the following form:
    # ?max_id=847960489447628799&q=%23RIPSelena&count=100&include_entities=1
    kwargs = dict([ kv.split('=') for kv in unquote(next_results[1:]).split("&") ])
    
    search_results = twitter_api.search.tweets(**kwargs)
    statuses += search_results['statuses']

print(json.dumps(statuses[0], indent=1))
for i in range(10):
    print()
    print(statuses[i]['text'])
    print('Favorites: ', statuses[i]['favorite_count'])
    print('Retweets: ', statuses[i]['retweet_count'])<jupyter_output>RT @Rosenchild: #DeepLearning Algorithm Diagnoses Skin #Cancer as well as Seasoned Dermatologists - #Healthcare, #Medicine - https://t.co/n…
Favorites:  0
Retweets:  4

RT @Rosenchild: Researchers Speeding Up Reporting of #Cancer Data with help from #ArtificialIntelligence and #DeepLearning, GPUs - https://…
Favorites:  0
Retweets:  10

RT @Rosenchild: #ArtificialIntelligence - #AI, #DeepLearning Drops Error Rate for Breast #Cancer Diagnoses by 85% - https://t.co/nrV8r9TBj5…
Favorites:  0
Retweets:  14

RT @reworkdl: <NAME> from @goTrueMotion on using #Deeplearning on messy #sensor data #reworkDL #data #device #vectors https://t.co/vD…
Favorites:  0
Retweets:  4

#JAPAN #STARTUP:Roundup of startup #funding. Read on to see what caught investors’ eyes..#DEEPLEARNING..#FINTECH… https://t.co/S2Ew3Yx61q
Favorites:  0
Retweets:  0

RT @craigbrownphd: #FinTech #DeepLearning #Cloud #BlockChain #CraigBrownPhD <NAME> became a registered member https://t.co/aFQhPQ1…
Favorites:  0
[...]<jupyter_text>Twitter friends<jupyter_code>#-----------------------------------------------------------------------
# this is the user whose friends we will list
#-----------------------------------------------------------------------
username = "nikbearbrown"

#-----------------------------------------------------------------------
# perform a basic search 
# twitter API docs: https://dev.twitter.com/rest/reference/get/friends/ids
#-----------------------------------------------------------------------
query = twitter_api.friends.ids(screen_name = username)

#-----------------------------------------------------------------------
# tell the user how many friends we've found.
# note that the twitter API will NOT immediately give us any more 
# information about friends except their numeric IDs...
#-----------------------------------------------------------------------
print ("found %d friends" % (len(query["ids"])))
#-----------------------------------------------------------------------
# now we loop through them to pull out more info, in blocks of 100.
#-----------------------------------------------------------------------
for n in range(0, len(query["ids"]), 100):
	ids = query["ids"][n:n+100]

	#-----------------------------------------------------------------------
	# create a subquery, looking up information about these users
	# twitter API docs: https://dev.twitter.com/rest/reference/get/users/lookup
	#-----------------------------------------------------------------------
	subquery = twitter_api.users.lookup(user_id = ids)

	for user in subquery:
		#-----------------------------------------------------------------------
		# now print out user info, starring any users that are Verified.
		#-----------------------------------------------------------------------
		print (" [%s] %s - %s" % ("*" if user["verified"] else " ", user["screen_name"], user["location"]))<jupyter_output>[ ] ChrisKuffert - Cambridge, MA
 [ ] JeremyShawaf - Los Angeles, CA
 [ ] iBlong2iYush - 
 [ ] eyekantbeme - Venice, CA
 [*] karpathy - Stanford
 [*] Medium - San Francisco, CA, US
 [*] AirbnbEng - San Francisco, CA
 [ ] VoiceWardrobe - 
 [ ] chris_mavian - 
 [*] waze - Worldwide
 [ ] BobsBurgersJon - 
 [*] TheScottAdkins - 
 [ ] StudiosMajin - Matthews, NC
 [*] iliza - Hollywood, CA
 [ ] angelaleemma - Hawaii X Singapore 
 [ ] esite_loveth - 
 [*] preetalina - NJ/NYC
 [*] davidgoggins - 
 [*] ferrarifoster - Tuscaloosa, AL
 [*] NicoleEggert - 
 [ ] NewtonCaylin - 
 [*] JJWatt - United States of America
 [ ] BarberEcology - 
 [ ] MartinRiese - Los Angeles
 [ ] juliusdavis32 - Menomonee Falls, WI
 [*] freddie - 
 [*] Dembouz - Barcelona, Spain
 [*] 247Sports - Your team. All the time.
 [*] owillis - Takoma Park, MD
 [ ] skynellor - PLANET EARTH
 [*] marcorubio - United States
 [ ] YesYoureRacist - A dark corner of the Internet
 [ ] BrewerWellness - Santa Monica, CA
 [ ] MauricioPena - [...]<jupyter_text>Twitter friendship<jupyter_code>source = "nikbearbrown"
target = "infoinaction"

#-----------------------------------------------------------------------
# perform the API query
# twitter API docs: https://dev.twitter.com/rest/reference/get/friendships/show
#-----------------------------------------------------------------------
result = twitter_api.friendships.show(source_screen_name = source,
                                  target_screen_name = target)

#-----------------------------------------------------------------------
# extract the relevant properties
#-----------------------------------------------------------------------
following = result["relationship"]["target"]["following"]
follows   = result["relationship"]["target"]["followed_by"]

print ("%s following %s: %s" % (source, target, follows))
print ("%s following %s: %s" % (target, source, following))<jupyter_output>nikbearbrown following infoinaction: True
infoinaction following nikbearbrown: True<jupyter_text>Twitter home timeline<jupyter_code>statuses = twitter_api.statuses.home_timeline(count = 50)
print (statuses)

#-----------------------------------------------------------------------
# loop through each of my statuses, and print its content
#-----------------------------------------------------------------------
for status in statuses:
	print ("(%s) @%s %s" % (status["created_at"], status["user"]["screen_name"], status["text"]))<jupyter_output>[{'created_at': 'Thu Oct 26 05:13:38 +0000 2017', 'id': 923417312883105793, 'id_str': '923417312883105793', 'text': 'RT @zacharywhyde: I wanna go to haunted maze but need someone to go with me so I can throw a body at anything that scares me', 'truncated': False, 'entities': {'hashtags': [], 'symbols': [], 'user_mentions': [{'screen_name': 'zacharywhyde', 'name': '☁︎', 'id': 2370842690, 'id_str': '2370842690', 'indices': [3, 16]}], 'urls': []}, 'source': '<a href="https://about.twitter.com/products/tweetdeck" rel="nofollow">TweetDeck</a>', 'in_reply_to_status_id': None, 'in_reply_to_status_id_str': None, 'in_reply_to_user_id': None, 'in_reply_to_user_id_str': None, 'in_reply_to_screen_name': None, 'user': {'id': 396560365, 'id_str': '396560365', 'name': 'film feels 🎞', 'screen_name': 'filmfeeIs', 'location': '', 'description': 'the art of films', 'url': None, 'entities': {'description': {'urls': []}}, 'protected': False, 'followers_count': 33467, 'friends_count': 8649, 'listed_count': [...]<jupyter_text>twitter list lists<jupyter_code>import pprint
users = [ "nikbearbrown", "kylekuzma", "AliSingerYoga" ]
for user in users:
	print ("@%s" % (user))

	#-----------------------------------------------------------------------
	# ...retrieve all of the lists they own.
	# twitter API docs: https://dev.twitter.com/rest/reference/get/lists/list
	#-----------------------------------------------------------------------
	result = twitter_api.lists.list(screen_name = user)
	for list in result:
		print (" - %s (%d members)" % (list["name"], list["member_count"]))<jupyter_output>@nikbearbrown
 - NFL Draft (4 members)
@kylekuzma
@AliSingerYoga<jupyter_text>Twitter list retweets<jupyter_code>user='nikbearbrown'
results = twitter_api.statuses.user_timeline(screen_name = user)

#-----------------------------------------------------------------------
# loop through each of my statuses, and print its content
#-----------------------------------------------------------------------
for status in results:
	print ("@%s %s" % (user, status["text"]))

	#-----------------------------------------------------------------------
	# do a new query: who has RT'd this tweet?
	#-----------------------------------------------------------------------
	retweets = twitter_api.statuses.retweets._id(_id = status["id"])
	for retweet in retweets:
		print (" - retweeted by %s" % (retweet["user"]["screen_name"]))<jupyter_output>@nikbearbrown Just posted a video @ Huntington Avenue YMCA https://t.co/W1oVa9znzx
@nikbearbrown Just posted a photo https://t.co/kgsHKNqwqK
@nikbearbrown Just posted a photo @ Huntington Avenue YMCA https://t.co/9p9oML5ia1
@nikbearbrown My dream car https://t.co/Krkg6vg3D4
@nikbearbrown UCLA ranked No. 1 U.S. public university by Times Higher Education https://t.co/e545X19KKc
@nikbearbrown Tesla Reveals ‘Flagship’ Puerto Rico Project, and It Looks Great by @mikearildbrown https://t.co/TdhNzOj6sR via @inversedotcom
@nikbearbrown Puerto Rico and Whitefish Energy defend controversial $300 million power contract https://t.co/LO1eh7gLfU via @bi_contributors
@nikbearbrown Tiny Montana Firm Gets $300 Million Contract To Help Restore Power In Puerto Rico https://t.co/NQQoqIOzkD
@nikbearbrown Just posted a video @ <NAME> Grill &amp; Tap https://t.co/NFC1XowHAL
@nikbearbrown RT @KDISAWARRIOR: <NAME> praying pregame with a kid who lost his father due to a car accident. 🙏🏽🙏🏽🙏🏽 https[...]<jupyter_text>Twitter search geo<jupyter_code># Northeastern University  42.3398° N, 71.0892° W
latitude = 42.3398	# geographical centre of search
longitude = -71.0892	# geographical centre of search
max_range = 1 			# search range in kilometres
num_results = 50		# minimum results to obtain

result_count = 0
last_id = None

while result_count <  num_results:
	#-----------------------------------------------------------------------
	# perform a search based on latitude and longitude
	# twitter API docs: https://dev.twitter.com/rest/reference/get/search/tweets
	#-----------------------------------------------------------------------
	query = twitter_api.search.tweets(q = "", geocode = "%f,%f,%dkm" % (latitude, longitude, max_range), count = 100, max_id = last_id)

	for result in query["statuses"]:
		#-----------------------------------------------------------------------
		# only process a result if it has a geolocation
		#-----------------------------------------------------------------------
		if result["geo"]:
			user = result["user"]["screen_name"]
			text = result["text"]
			text = text.encode('ascii', 'replace')
			latitude = result["geo"]["coordinates"][0]
			longitude = result["geo"]["coordinates"][1]

			#-----------------------------------------------------------------------
			# now write this row to our CSV file
			#-----------------------------------------------------------------------
			row = [ user, latitude, longitude, text ]
			print ("%s %s %s"% (row[0],row[2],row[2]))
			print ("%s"% (row[3]))
			result_count += 1
		last_id = result["id"]

	#-----------------------------------------------------------------------
	# let the user know where we're up to
	#-----------------------------------------------------------------------
	print ("got %d results" % result_count)<jupyter_output>LloydLegacy617 -71.0889541 -71.0889541
b'This freaking fellow gonna pay me compliments like this on HIS birthday ....WHAT AN EPIC GUY ?? https://t.co/kXJoSuYeHy'
BrewhoundMa -71.0838799 -71.0838799
b'#limboipa from @longtrailbeer . Sexy beer to drink with an even sexier woman to drink it with.? https://t.co/rKR7J6Toa9'
JohnMichaelVibe -71.0870288 -71.0870288
b'Here?s a short clip from tonight?s rehearsal for @juliagartha Caf? https://t.co/pMHcLcfgpV'
BrewhoundMa -71.08565 -71.08565
b'Enjoying a Super Session #8 by @lawsonsfinest . Great start to this long weekend. #beer? https://t.co/eENwt0zW6a'
BrewhoundMa -71.0857 -71.0857
b'Drinking a Conehead by @ZeroGravityBeer at @bukowskiboston ? https://t.co/GVXPO5Pq7e #photo'
BrewhoundMa -71.0857 -71.0857
b'Drinking a Super Session #8 by @LawsonsFinest at @bukowskiboston ? https://t.co/Bx59K1PaCi #photo'
tozzi5284 -71.09452 -71.09452
b'a night at mfaboston ? @ Museum of Fine Arts, Boston https://t.co/G476amIqJF'
saritaforrest -71.09452 -71.094[...]<jupyter_text>Extracting text, screen names, and hashtags from tweets<jupyter_code>status_texts = [ status['text'] 
                 for status in statuses ]

screen_names = [ user_mention['screen_name'] 
                 for status in statuses
                     for user_mention in status['entities']['user_mentions'] ]

hashtags = [ hashtag['text'] 
             for status in statuses
                 for hashtag in status['entities']['hashtags'] ]

# Compute a collection of all words from all tweets
words = [ w 
          for t in status_texts 
              for w in t.split() ]

# Explore the first 5 items for each...

print(json.dumps(status_texts[0:5], indent=1))
print(json.dumps(screen_names[0:5], indent=1) )
print(json.dumps(hashtags[0:5], indent=1))
print(json.dumps(words[0:5], indent=1))<jupyter_output>[
 "RT @Rosenchild: #DeepLearning Algorithm Diagnoses Skin #Cancer as well as Seasoned Dermatologists - #Healthcare, #Medicine - https://t.co/n\u2026",
 "RT @Rosenchild: Researchers Speeding Up Reporting of #Cancer Data with help from #ArtificialIntelligence and #DeepLearning, GPUs - https://\u2026",
 "RT @Rosenchild: #ArtificialIntelligence - #AI, #DeepLearning Drops Error Rate for Breast #Cancer Diagnoses by 85% - https://t.co/nrV8r9TBj5\u2026",
 "RT @reworkdl: <NAME> from @goTrueMotion on using #Deeplearning on messy #sensor data #reworkDL #data #device #vectors https://t.co/vD\u2026",
 "#JAPAN #STARTUP:Roundup of startup #funding. Read on to see what caught investors\u2019 eyes..#DEEPLEARNING..#FINTECH\u2026 https://t.co/S2Ew3Yx61q"
]
[
 "Rosenchild",
 "Rosenchild",
 "Rosenchild",
 "reworkdl",
 "goTrueMotion"
]
[
 "DeepLearning",
 "Cancer",
 "Healthcare",
 "Medicine",
 "Cancer"
]
[
 "RT",
 "@Rosenchild:",
 "#DeepLearning",
 "Algorithm",
 "Diagnoses"
]<jupyter_text>Frequency distribution from the words in tweets<jupyter_code>from collections import Counter

for item in [words, screen_names, hashtags]:
    c = Counter(item)
    print(c.most_common()[:10]) # top 10
    print()
from prettytable import PrettyTable

for label, data in (('Word', words), 
                    ('Screen Name', screen_names), 
                    ('Hashtag', hashtags)):
    pt = PrettyTable(field_names=[label, 'Count']) 
    c = Counter(data)
    [ pt.add_row(kv) for kv in c.most_common()[:10] ]
    pt.align[label], pt.align['Count'] = 'l', 'r' # Set column alignment
    print(pt)<jupyter_output>+------------------+-------+
| Word             | Count |
+------------------+-------+
| #DeepLearning    |   258 |
| RT               |   240 |
| #AI              |   155 |
| #MachineLearning |   131 |
| #BigData         |   129 |
| to               |    72 |
| of               |    64 |
| #DataScience     |    62 |
| #BlockChain      |    51 |
| a                |    51 |
+------------------+-------+
+-----------------+-------+
| Screen Name     | Count |
+-----------------+-------+
| craigbrownphd   |    35 |
| DeepLearn007    |    20 |
| MikeQuindazzi   |    17 |
| KirkDBorne      |    16 |
| andi_staub      |    12 |
| SachinLulla     |    12 |
| JacBurns_Comext |    11 |
| nodexl          |    11 |
| alevergara78    |    11 |
| evankirstel     |    11 |
+-----------------+-------+
+-----------------+-------+
| Hashtag         | Count |
+-----------------+-------+
| DeepLearning    |   279 |
| AI              |   176 |
| MachineLearning |   133 |
| BigData         |   130 |
| Data[...]<jupyter_text>Calculating lexical diversity for tweets<jupyter_code>def lexical_diversity(tokens):
    return len(set(tokens))/len(tokens) 

def average_words(statuses):
    total_words = sum([ len(s.split()) for s in statuses ]) 
    return total_words/len(statuses)

print(lexical_diversity(words))
print(lexical_diversity(screen_names))
print(lexical_diversity(hashtags))
print(average_words(status_texts))<jupyter_output>0.23083759164630083
0.3111111111111111
0.08519900497512438
13.639393939393939<jupyter_text>Most popular retweets<jupyter_code>retweets = [
            # Store out a tuple of these three values ...
            (status['retweet_count'], 
             status['retweeted_status']['user']['screen_name'],
             status['retweeted_status']['id'],
             status['text']) 
            
            # ... for each status ...
            for status in statuses 
            
            # ... so long as the status meets this condition.
                if 'retweeted_status' in status.keys()
           ]

# Slice off the first 5 from the sorted results and display each item in the tuple

pt = PrettyTable(field_names=['Count', 'Screen Name', 'Tweet ID', 'Text'])
[ pt.add_row(row) for row in sorted(retweets, reverse=True)[:5] ]
pt.max_width['Text'] = 50
pt.align= 'l'
print(pt)<jupyter_output>+-------+--------------+--------------------+----------------------------------------------------+
| Count | Screen Name  | Tweet ID           | Text                                               |
+-------+--------------+--------------------+----------------------------------------------------+
| 242   | DeepLearn007 | 923017972800741383 | RT @DeepLearn007: An AI Found Dozens of            |
|       |              |                    | Gravitational Lens Galaxies                        |
|       |              |                    | #AI #MachineLearning #DeepLearning #BigData #ML    |
|       |              |                    | #DL #Science #tech                                 |
|       |              |                    | http…                                              |
| 242   | DeepLearn007 | 923017972800741383 | RT @DeepLearn007: An AI Found Dozens of            |
|       |              |                    | Gravitational Lens Galaxies                        |
|       | [...]<jupyter_text>Users who have retweeted a status<jupyter_code># Get the original tweet id for a tweet from its retweeted_status node 
# and insert it here

_retweets = twitter_api.statuses.retweets(id=922562320080953344)
print([r['user']['screen_name'] for r in _retweets])<jupyter_output>['UwillBAssessed', 'zippylab', 'machinelearnbot', 'alevergara78', 'PaulStechai', 'yanstv', 'IoTimelab', 'cmarmulla', 'habitline', 'UtarSystems', 'TheTuringSpirit', 'ray_kuersten', 'DarcyOyewoleWeb', 'alevergara78', 'sulefati7', 'savia_digital', 'amalmerzouk', 'z2duqOb3i6eJzMO', 'EricTIXADOR', 'KoffaO']<jupyter_text>Sentiment Analysis<jupyter_code># pip install nltk
import nltk
nltk.download('vader_lexicon')

import numpy as np
from nltk.sentiment.vader import SentimentIntensityAnalyzer
twitter_stream = twitter.TwitterStream(auth=auth)
iterator = twitter_stream.statuses.sample()
tweets = []
for tweet in iterator:
    try:
        if tweet['lang'] == 'en':
            tweets.append(tweet)
    except:
        pass
    if len(tweets) == 100:
        break
analyzer = SentimentIntensityAnalyzer()
analyzer.polarity_scores('Hi')
analyzer.polarity_scores('Happy birthday to the loveliest lady. I really like like you.')
analyzer.polarity_scores('We messed up tonight... bad 🙃.')
scores = np.zeros(len(tweets))

for i, t in enumerate(tweets):
    # Extract the text portion of the tweet
    text = t['text']
    
    # Measure the polarity of the tweet
    polarity = analyzer.polarity_scores(text)
    
    # Store the normalized, weighted composite score
    scores[i] = polarity['compound']
most_positive = np.argmax(scores)
most_negative = np.argmin(scores)
print('{0:6.3f} : "{1}"'.format(scores[most_positive], tweets[most_positive]['text']))
print('{0:6.3f} : "{1}"'.format(scores[most_negative], tweets[most_negative]['text']))<jupyter_output>-0.772 : "RT @yashar: <NAME> Gets Death Threats From White Nationalist https://t.co/Ayn6IfSPXb"
{'fraction_non_alphanumeric': 0.23874151758173967, 'fraction_numerical': 0.030668899268529125, 'mean_word_length': 2.239366257493577, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 60, 'https://': 40, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '355715', 'license': 'MIT', 'n_tokens_mistral': 7594, 'n_tokens_neox': 6818, 'n_words': 1754, 'path': 'Week_2/NBB_Social_Web_Twitter.ipynb', 'repo_name': 'nikbearbrown/INFO_6210', 'size': '22799'}