<jupyter_start><jupyter_text>Twitter streaming API<jupyter_code>import tweepy, json

# Store OAuth authentication credentials in relevant variables
access_token = "<KEY>"
access_token_secret = "<KEY>"
consumer_key = "qzl18bdauXfSVsfJWLcikvxqJ"
consumer_secret = "<KEY>"

# Pass OAuth details to tweepy's OAuth handler
auth = tweepy.OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_token_secret)
class MyStreamListener(tweepy.StreamListener):
    def __init__(self, api=None):
        super(MyStreamListener, self).__init__()
        self.num_tweets = 0
        self.file = open("tweets.txt", "w")

    def on_status(self, status):
        tweet = status._json
        self.file.write( json.dumps(tweet) + '\n' )
        self.num_tweets += 1
        if self.num_tweets < 100:
            return True
        else:
            return False
        self.file.close()

    def on_error(self, status):
        print(status)
# Initialize Stream listener
l = MyStreamListener()

# Create Stream object with authentication
stream = tweepy.Stream(auth,l)

# Filter Twitter Streams to capture data by keywords; does capitalization matter?
stream.filter(track = ['federer','nadal','djokovic'])
# Initialize empty list to store tweets
tweets_data = []

# Open connection to file, or could do context manager
tweets_file = open('tweets.txt', "r")

# Read in tweets and store in list
for line in tweets_file:
    tweet = json.loads(line)
    tweets_data.append(tweet)

# Close connection to file
tweets_file.close()

# Print the keys of json/dict for first tweet
print(tweets_data[0].keys())
import pandas as pd

# Build DataFrame of tweet texts, users, and times
tweets_df = pd.DataFrame(tweets_data, columns=['text','created_at','lang'])

print(tweets_df.head())
print(tweets_df.info())
import re

def word_in_text(word, tweet):
    word = word.lower()
    text = tweet.lower()
    match = re.search(word, tweet)

    if match:
        return True
    else:
        return False
# Initialize list to store tweet counts
[federer, nadal, djokovic] = [0, 0, 0]

# Iterate through tweets_df, counting the number of tweets in which each person is mentioned
for index, row in tweets_df.iterrows():
    federer += word_in_text('Federer', row['text'])
    nadal += word_in_text('Nadal', row['text'])
    djokovic += word_in_text('Djokovic', row['text'])
    
print([federer, nadal, djokovic])
import seaborn as sns
import matplotlib.pyplot as plt

# Set seaborn style
sns.set(color_codes=True)

# Create a list of labels
cd = ['federer', 'nadal', 'djokovic']

# Plot histogram
ax = sns.barplot(cd,[federer, nadal, djokovic])
ax.set(ylabel="number of tweets")
plt.show()
import requests
url = 'http://export.arxiv.org/api/query?search_query=all:electron&start=0&max_results=1'
# Package the request, send the request and catch the response
r = requests.get(url)

# Print the (xml) text of the response
# print(r.text)
import xmltodict, json

o = xmltodict.parse(r.text)
j_string = json.dumps(o)
j_file = json.loads(j_string)
print(j_file['feed']['entry']['title'])
url = 'http://export.arxiv.org/api/query?search_query=cat:math.AG&start=0&max_results=2'
# Package the request, send the request and catch the response
r = requests.get(url)
o = xmltodict.parse(r.text)
j_string = json.dumps(o)
j_file = json.loads(j_string)
print(j_file['feed']['entry'])<jupyter_output>[{'id': 'http://arxiv.org/abs/math/9204225v1', 'updated': '1992-04-01T00:00:00Z', 'published': '1992-04-01T00:00:00Z', 'title': 'Higgs line bundles, Green-Lazarsfeld sets,and maps of Kähler manifolds\n  to curves', 'summary': 'Let $X$ be a compact K\\"ahler manifold. The set $\\cha(X)$ of one-dimensional\ncomplex valued characters of the fundamental group of $X$ forms an algebraic\ngroup. Consider the subset of $\\cha(X)$ consisting of those characters for\nwhich the corresponding local system has nontrivial cohomology in a given\ndegree $d$. This set is shown to be a union of finitely many components that\nare translates of algebraic subgroups of $\\cha(X)$. When the degree $d$ equals\n1, it is shown that some of these components are pullbacks of the character\nvarieties of curves under holomorphic maps. As a corollary, it is shown that\nthe number of equivalence classes (under a natural equivalence relation) of\nholomorphic maps, with connected fibers, of $X$ onto smooth curves of a [...]<jupyter_text>ArXiv API: parsing and paging<jupyter_code>import urllib.request
import feedparser
# Base api query url
base_url = 'http://export.arxiv.org/api/query?'

# Search parameters
search_query = 'cat:math.AP'
start = 0                     # retrieve the first 10 results
max_results = 10

query = 'search_query=%s&start=%i&max_results=%i' % (search_query, start, max_results)
# perform a GET request using the base_url and query
# response = requests.get(base_url+query)

response = urllib.request.urlopen(base_url+query)

# parse the response using feedparser
feed = feedparser.parse(response)
for entry in feed.entries:
#    print('e-print metadata')
#    print 'arxiv-id: %s' % entry.id.split('/abs/')[-1]
#    print 'Published: %s' % entry.published
    print('Title: %s' % entry.title)
import time

# Base api query url
base_url = 'http://export.arxiv.org/api/query?'

# Search parameters
search_query = 'cat:math.AG'
start = 0                       # start at the first result
total_results = 500              # want 20 total results
results_per_iteration = 25       # 5 results at a time
wait_time = 3                   # number of seconds to wait between calls

print('Searching arXiv for %s' % search_query)

titles_list = []

for i in range(start,total_results,results_per_iteration):
    
#    print("Results %i - %i" % (i,i+results_per_iteration))
    
    query = 'search_query=%s&start=%i&max_results=%i' % (search_query,
                                                         i,
                                                        results_per_iteration)

    # perform a GET request using the base_url and query
    response = urllib.request.urlopen(base_url+query)

    # parse the response using feedparser
    feed = feedparser.parse(response)

    # Run through each entry, and print out title
    for entry in feed.entries:
#        print 'arxiv-id: %s' % entry.id.split('/abs/')[-1]
#        print(entry.title)
        titles_list.append(entry.title.replace('\n ',''))
    
    # Remember to play nice and sleep a bit before you call the api again!
#    print('Sleeping for %i seconds' % wait_time)
    time.sleep(wait_time)
print(titles_list)
with open('titles_test.csv','w') as file:
    for title in titles_list:
        file.write(title)
        file.write('\n')
import pandas as pd

titles_df = pd.read_csv('titles_test.csv', header = None, names = ['title'])

print(titles_df)<jupyter_output>title
0   Some non-analytic-hypoelliptic sums of squares...
1   A steepest descent method for oscillatory Riem...
2                           Semilinear wave equations
3   A sharp pointwise bound for functions with $L^...
4   user's guide to viscosity solutions of second ...
5   Smooth static solutions of the Einstein-Yang/M...
6   A new result for the porous medium equation de...
7   A Formula for Finding a Potential from Nodal L...
8   Local Solvability For a Class of Partial Diffe...
9   Global Irregularity For Degenerate Elliptic Op...
10  Infinite dimensional families of locally nonso...
11  The monodromy matrix for a family of almost pe...
12          Low regularity semi-linear wave equations
13  Variational evolution problems and nonlocal ge...
14  Complete Integrability of Completely Integrabl...
15  Action-Angle variables for the Gel'fand-Dikii ...
16  Transition operators of diffusions reduce zero...
17  On a singular limit prob[...]
{'fraction_non_alphanumeric': 0.10942522823710943, 'fraction_numerical': 0.012215507265012216, 'mean_word_length': 3.651913875598086, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '438890', 'license': 'MIT', 'n_tokens_mistral': 2566, 'n_tokens_neox': 2417, 'n_words': 774, 'path': 'API practice.ipynb', 'repo_name': 'kadakia/API-practice', 'size': '7907'}