<jupyter_start><jupyter_text>By @<NAME> Extraction of tweets against the 'URL' of scientific papers.<jupyter_code>#!/Users/christine/anaconda3/bin/python
# -*- coding: utf-8 -*-

import os, re, pandas as pd, time
import tweepy as tw
from tweepy.auth import OAuthHandler

consumer_key = ''
consumer_secret = ''
access_token_key = ''
access_token_secret = ''

auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token_key, access_token_secret)
api = tw.API(auth, wait_on_rate_limit=True)

# SEARCH TWEETS
def search_article(query):
    query = '"'+query+'"'
    try:
        # get 999 tweets since Oct. 1 2019, untruncated
        tweets = tw.Cursor(api.search, q=query, since="2019-10-01", tweet_mode='extended').items(999)
        tweets = [tweet.full_text for tweet in tweets]
        print(len(tweets), query) # progress
        return tweets
    except:
        print('ERROR', query)
        return []

# GET LIST OF ARTICLES
df = pd.read_csv('articles_vaccine_covid2019.csv', encoding='utf-8-sig')
links = df['URL for article']
titles = df['Article name']
titles = [re.sub(r' *\[.*\] *', '', t) for t in titles] # remove stuff in brackets
articles = [(titles[i], links[i]) for i in range(len(links))] # [(title, link), ...]

# SEARCH FOR ARTICLE MENTIONS ON TWITTER
tweets, article_titles, article_links, found_by = [], [], [], []
i = 0
for title, link in articles[i:]:
    # search for article title
    t = search_article(title)
    tweets.extend(t)
    article_titles.extend([title]*len(t))
    article_links.extend([link]*len(t))
    found_by.extend(['article_title']*len(t))
    
    # search for article link
    t = search_article('url:'+link)
    tweets.extend(t)
    article_titles.extend([title]*len(t))
    article_links.extend([link]*len(t))
    found_by.extend(['article_link']*len(t))
    # checkpoint save
    if i % 10 == 0:
        df = pd.DataFrame({'tweet':tweets, 'article_title':article_titles, 'article_link':article_links, 'found_by':found_by})
        df = df.drop_duplicates()
        df.to_csv('result.csv', index=False, encoding='utf-8-sig')
    print(i) # progress
    i += 1
    time.sleep(10) # wait a bit

# SAVE RESULTS TO CSV
print(len(tweets))
df = pd.DataFrame({'tweet':tweets, 'article_title':article_titles, 'article_link':article_links, 'found_by':found_by})
df = df.drop_duplicates()
df.to_csv('VT_tweets.csv', index=False, encoding='utf-8-sig')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15331152902698283, 'fraction_numerical': 0.014309076042518397, 'mean_word_length': 4.273706896551724, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '617818', 'license': 'MIT', 'n_tokens_mistral': 917, 'n_tokens_neox': 866, 'n_words': 185, 'path': "twitter_sentiment/Tweets_extraction_against_articles'_URL.ipynb", 'repo_name': 'yangjunqingqiao/task-vt', 'size': '2454'}