<jupyter_start><jupyter_text>Twitter Followers Ego Graph Scraper | `twint`Since `twint` is not offically supported by Twitter API, Twitter will interrupt follower stream without any notification after an unspecified time interval.The scraper will go on without excepting, effectively resuming to download users only when Twitter re-allows it to do so, which causes a great portions of followers to ultimatly be missing . It is thus difficult to obtain large egographs with scapers like `twint`.For further information see the [documentation](https://github.com/twintproject/twint/wiki).<jupyter_code>import twint
import nest_asyncio
import pandas as pd
from progressbar import ProgressBar, Bar, Percentage
import time
nest_asyncio.apply()

def fso(who, whose, limit = None):
    c = twint.Config()
    c.Username = whose
    # later we want to retrieve followers as a pandas dataframe
    c.Pandas = True
    if limit is not None:
        c.Limit = limit
    c.Hide_output = True 
    
    try: 
        if who == "followers":
            twint.run.Followers(c)
        elif who == "following":
            print("following")
            twint.run.Following(c)
    except:
        print("an error occurred")
            
    fo_df = twint.storage.panda.Follow_df
    fo_list = fo_df[who][whose]
    print("len(fo.list) = ", len(fo_list))
    
    # get second order
    so_df = pd.DataFrame() 
    maxval = len(fo_list[:10]) +1
    i = 0
    pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=maxval).start()
    for user in fo_list[:10]:
        try:
            time.sleep(10)
            c = twint.Config()
            c.Username = user
            c.Pandas = True
            if limit is not None:
                c.Limit = limit        
            c.Hide_output = True 

            if who == "followers":
                twint.run.Followers(c)
            elif who == "following":
                twint.run.Following(c)
            so_temp = twint.storage.panda.Follow_df
            print(so_temp, "type(so_temp) = ", type(so_temp))
            so_df = so_df.append(so_temp)
        except:
            print("an error occurred")
            
        i = i+1
        pbar.update(i)
    pbar.finish()
    return (fo_df,so_df)
fo_df, so_df = fso(who = "following",whose = "PietroMonticone", limit = 20 )
# fo_df.to_csv(r"../working/fo_ego_pietro_20.csv")
# so_df.to_csv(r"../working/so_ego_pietro_20.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10093992644053944, 'fraction_numerical': 0.006129955046996322, 'mean_word_length': 2.395284327323162, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '484204', 'license': 'MIT', 'n_tokens_mistral': 791, 'n_tokens_neox': 753, 'n_words': 209, 'path': 'Scrapers/Twint/Twint_TwitterFollowersEgographScraper.ipynb', 'repo_name': 'daorse/EgoGraphProject', 'size': '2447'}