<jupyter_start><jupyter_text>Twitter constants configuration<jupyter_code>CONSUMER_KEY = "wdCgGygFbK3mwuiPebsdbZG08"
CONSUMER_SECRET = "<KEY>"

ACCESS_TOKEN_KEY = "<KEY>"
ACCESS_TOKEN_SECRET = "<KEY>"
AUTH = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
AUTH.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)

API = tweepy.API(AUTH)<jupyter_output><empty_output><jupyter_text>Set words for removing<jupyter_code>words2remove = list(set(stopwords.words("english")))
more_words2remove = [
    "#", ":", "@",
    "href", "https",
    "(", ")", "...", ".",
    "rt", "``", ",", "''",
    "-", "?", "!", "'s",
    "'i", "'", "|", "//", ";",
    "&", "%", "[", "]", "{", "}",
    "`", "’", "n't"
]

words2remove += more_words2remove

emoji_pattern = re.compile(
    "[" u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
    "]+", flags=re.UNICODE)<jupyter_output><empty_output><jupyter_text>Define API<jupyter_code>def clean(text):
    tokens = nltk.word_tokenize(text.lower())
    clean = ""

    for x in tokens:
        if x not in words2remove:
            clean += x
            clean += " "
        
    return clean

def get_tweets(team, count):
    answer = []
    results = API.search(q="2018 World Cup "+str(team), lang="en", count=count)
    for result in results:
        answer.append(clean(result.text))
    
    return answer
get_tweets("Brazil", 30)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18983268983268983, 'fraction_numerical': 0.039897039897039896, 'mean_word_length': 3.7264437689969605, 'pattern_counts': {'":': 1, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '620810', 'license': 'MIT', 'n_tokens_mistral': 656, 'n_tokens_neox': 604, 'n_words': 84, 'path': 'TwitterAPI/main.ipynb', 'repo_name': 'VIVelev/WorldCup-Prediction', 'size': '1684'}