<jupyter_start><jupyter_text>• Choose a popular Twitter account (≥ 10, 000 followers).• Build the friendship graph for that account’s followers.• Choose a method of community detection and apply it to that graph.• Produce a visual display of your graph such that nodes belonging to different communities are differently colored.<jupyter_code>import json
import pandas as pd
from twitter import *
from numpy import random<jupyter_output><empty_output><jupyter_text>Replace twitter access token and other infomartion in associated auth_dict file<jupyter_code># Loading my authentication tokens
with open('auth_dict','r') as f:
    twtr_auth = json.load(f)

t = Twitter(auth=OAuth(twtr_auth['token'], twtr_auth['token_secret'], 
            twtr_auth['consumer_key'], twtr_auth['consumer_secret']))
import sys
import time
from twitter.api import TwitterHTTPError
from urllib.error import URLError
from http.client import BadStatusLine

def make_twitter_request(twitter_api_func, max_errors=10, *args, **kw):
    # A nested helper function that handles common HTTPErrors. Return an updated
    # value for wait_period if the problem is a 500 level error. Block until the
    # rate limit is reset if it's a rate limiting issue (429 error). Returns None
    # for 401 and 404 errors, which requires special handling by the caller.
    def handle_twitter_http_error(e, wait_period=2, sleep_when_rate_limited=True):
        if wait_period > 3600: # Seconds
            print('Too many retries. Quitting.', file=sys.stderr)
            raise e
        if e.e.code == 401:
            return None
        elif e.e.code == 404:
            print('Encountered 404 Error (Not Found)', file=sys.stderr)
            return None
        elif e.e.code == 429:
            print('Encountered 429 Error (Rate Limit Exceeded)', file=sys.stderr)
            if sleep_when_rate_limited:
                print("Retrying in 15 minutes...ZzZ...", file=sys.stderr)
                sys.stderr.flush()
                time.sleep(60*15 + 5)
                print('...ZzZ...Awake now and trying again.', file=sys.stderr)
                return 2
            else:
                raise e # Caller must handle the rate limiting issue
        elif e.e.code in (500, 502, 503, 504):
            print('Encountered %i Error. Retrying in %i seconds' % (e.e.code, wait_period), file=sys.stderr)
            time.sleep(wait_period)
            wait_period *= 1.5
            return wait_period
        else:
            raise e

    # End of nested helper function

    wait_period = 2
    error_count = 0
    while True:
        try:
            return twitter_api_func(*args, **kw)
        except TwitterHTTPError as e:
            error_count = 0
            wait_period = handle_twitter_http_error(e, wait_period)
            if wait_period is None:
                return
        except URLError as e:
            error_count += 1
            print("URLError encountered. Continuing.", file=sys.stderr)
            if error_count > max_errors:
                print("Too many consecutive errors...bailing out.", file=sys.stderr)
                raise
        except BadStatusLine as e:
            error_count += 1
            print >> sys.stderr, "BadStatusLine encountered. Continuing."
            if error_count > max_errors:
                print("Too many consecutive errors...bailing out.", file=sys.stderr)
                raise
# This will let us create new partial
# functions with arguments set to 
# certain values.
from functools import partial

# This was maxint.
# There is no longer a maxint (in Python 3)
from sys import maxsize


def get_friends_followers_ids(twitter_api, screen_name=None, user_id=None,
                                friends_limit=maxsize, followers_limit=maxsize):
    # Must have either screen_name or user_id (logical xor)
    assert (screen_name != None) != (user_id != None), \
    "Must have screen_name or user_id, but not both"
    
    # You can also do this with a function closure.
    get_friends_ids = partial(make_twitter_request, twitter_api.friends.ids,
                                count=5000)
    get_followers_ids = partial(make_twitter_request, twitter_api.followers.ids,
                                count=5000)
    friends_ids, followers_ids = [], []
    for twitter_api_func, limit, ids, label in [
            [get_friends_ids, friends_limit, friends_ids, "friends"],
            [get_followers_ids, followers_limit, followers_ids, "followers"]
            ]:
        #LOOK HERE! This little line is important.
        if limit == 0: continue
        cursor = -1
        while cursor != 0:
            # Use make_twitter_request via the partially bound callable...
            if screen_name:
                response = twitter_api_func(screen_name=screen_name, cursor=cursor)
            else: # user_id
                response = twitter_api_func(user_id=user_id, cursor=cursor)
            if response is not None:
                ids += response['ids']
                cursor = response['next_cursor']
            print('Fetched {0} total {1} ids for {2}'.format(len(ids),
                    label, (user_id or screen_name), file=sys.stderr))
            if len(ids) >= limit or response is None:
                break
    # Do something useful with the IDs, like store them to disk...
    return friends_ids[:friends_limit], followers_ids[:followers_limit]

def get_user_profile(twitter_api, screen_names=None, user_ids=None):
    # Must have either screen_name or user_id (logical xor)
    assert (screen_names != None) != (user_ids != None), \
        "Must have screen_names or user_ids, but not both"
    items_to_info = {}
    items = screen_names or user_ids
    sorted_by_follow_count = []
    while len(items) > 0:
        # Process 100 items at a time per the API specifications for /users/lookup.
        # See https://dev.twitter.com/docs/api/1.1/get/users/lookup for details.
        items_str = ','.join([str(item) for item in items[:100]])
        items = items[100:]
        if screen_names:
            response = make_twitter_request(twitter_api.users.lookup, screen_name=items_str)
        else: # user_ids
            response = make_twitter_request(twitter_api.users.lookup, user_id=items_str)
        for user_info in response:
            if screen_names:
                items_to_info[user_info['followers_count']] = user_info['screen_name']
            else: # user_ids
                items_to_info[user_info['followers_count']] = user_info['id']
        #print("RESPONCE", response)
        #print("DISCTIONAY", items_to_info)
    for key in sorted(items_to_info, reverse=True):
        sorted_by_follow_count.append(items_to_info[key])
    return sorted_by_follow_count[:5]
# Create a mostly empty data frame,
# and write it to a CSV file.
df = pd.DataFrame(columns=['ID','ReciprocalFriend'])
df.to_csv('ReciprocalFriend.csv', index=False)

# Our function
def save_followers(fid, reciprocal_friend):
    data_frame_rf = [[str(fid), str(i)] for i in reciprocal_friend]
    #print(data_frame_rf)
    df = pd.DataFrame(data_frame_rf, columns=['ID','ReciprocalFriend'])
    with open('ReciprocalFriend.csv', 'a') as f:
        df.to_csv(f,header=False, index=False)<jupyter_output><empty_output><jupyter_text>Apply Breath First Search to generate friendship graphEach time we are selecting 100 reciprocal friends and finding top 5 friends with highest follower count.  These 5 reciprocal friends will be used for next level of Breadth-First search.<jupyter_code>def crawl_followers(twitter_api, screen_name, limit=1000000, depth=2):
    
    # Resolve the ID for screen_name and start working with IDs for consistency
    seed_id = str(twitter_api.users.show(screen_name=screen_name)['id'])
    friends_ids, followers_ids = get_friends_followers_ids(twitter_api, user_id=seed_id,
                                 friends_limit=limit, followers_limit=limit)
    rp_friend = list(set(friends_ids) & set(followers_ids))
    top_five = get_user_profile(t, user_ids=rp_friend[:100])
    next_queue = top_five
    # Store a seed_id => _follower_ids mapping in MongoDB
    
    save_followers(seed_id, next_queue)
    
    d = 1
    # Note that in the example in the next cell,
    # we never enter this loop.
    while d < depth:
        print("Number of ", d,"- Distance node", len(next_queue))
        d += 1
        # Reset the next_queue so that we can
        # start building up the next level
        # of followers-of-followers
        (queue, next_queue) = (next_queue, [])
        # Loop through the current
        # level of followers
        for fid in queue:
            friends_ids, followers_ids = get_friends_followers_ids(twitter_api, user_id=fid,
                                friends_limit=limit, followers_limit=limit)
            # Store an ID with a string recording
            # IDs of followers of the user with ID "fid"
            rp_friend = list(set(friends_ids) & set(followers_ids))
            if (len(rp_friend) == 0):
                continue
            top_five = get_user_profile(t, user_ids=rp_friend[-100:])
            save_followers(str(fid), top_five)
            # Extending the list
            next_queue += top_five
screen_name = 'timoreilly'
crawl_followers(t, screen_name, depth=4, limit=5000)<jupyter_output>Encountered 429 Error (Rate Limit Exceeded)
Retrying in 15 minutes...ZzZ...
...ZzZ...Awake now and trying again.<jupyter_text>Read graph stored in a file from the earlier step and generate networkx graph.<jupyter_code>import numpy as np
import pandas as pd
import networkx as nx
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
# Create Graph from file
df = pd.read_csv("ReciprocalFriend.csv")
x_point = list(df[df.columns[0]].values)
y_point = list(df[df.columns[1]].values)
edges_list = []
for i in range(len(x_point)):
    edges_list.append((x_point[i], y_point[i]))
node_list = set(x_point+y_point)
RG = nx.Graph()
RG.add_nodes_from(node_list)
RG.add_edges_from(edges_list)
# Display some graph information such as number of nodes and edges
print ("Number of Nodes :", RG.number_of_nodes())
print ("Numebr of edges :", RG.number_of_edges())<jupyter_output>Number of Nodes : 384
Numebr of edges : 405<jupyter_text>Apply Spectral clustering on the graph to detect communities<jupyter_code># find Laplacian matrix
A = nx.adjacency_matrix(RG)
D = np.diag(list(dict(nx.degree(RG)).values()))
L = D - A
# find required eigen value and eigen vector
lmbds,vctrs = np.linalg.eig(L)
indx = [i for i in range(len(lmbds)) if lmbds[i] > .01 and lmbds[i] < .03]
RG_mbd = vctrs[:,indx]
print ("We will devide our Graph in", len(indx), "communities")
# Apply k-means
est = KMeans(n_clusters=len(indx))
est.fit(RG_mbd)

# Labels will give us community number for each node
# we will use same community number as color value for each node
labels = est.labels_<jupyter_output>C:\Users\Bhushan\Anaconda3\lib\site-packages\sklearn\utils\validation.py:433: ComplexWarning: Casting complex values to real discards the imaginary part
  array = np.array(array, dtype=dtype, order=order, copy=copy)<jupyter_text>Display graph with the different color for each detected community<jupyter_code># Display Graph with different colors for each community
pos = nx.spring_layout(RG)
nx.draw_networkx_nodes(RG,pos,
                       nodelist=node_list,
                       node_color=labels,
                       node_size=100,
                   alpha=0.8)
nx.draw_networkx_edges(RG,pos)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09145232527929333, 'fraction_numerical': 0.012730579371265264, 'mean_word_length': 2.9039891818796484, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '340725', 'license': 'MIT', 'n_tokens_mistral': 3620, 'n_tokens_neox': 3407, 'n_words': 1016, 'path': 'src/CommunityDetection.ipynb', 'repo_name': 'Bhushan-Jagtap-2013/Community-Detection-in-Graphs', 'size': '11547'}