<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;DSPT6 - Unit 3 Module 2 - Consuming Data from an API2&nbsp;&nbsp;SpaCy Embeddings3&nbsp;&nbsp;Bringing it all together  DSPT6 - Unit 3 Module 2 - Consuming Data from an APIThe purpose of this notebook is to demonstrate:- Connect to the Twitter API (and twitter_scraper) to query for tweets and user information by various parameters- Convert tweet text using SpaCy into numerical embeddings that can be use in a predictive model<jupyter_code>import tweepy
# ADD YOUR CREDENTIALS HERE
TWITTER_CONSUMER_API_KEY = 'Ljnwq5zICyVT56UxRqzytWB7v'
TWITTER_CONSUMER_API_SECRET = '<KEY>'
TWITTER_ACCESS_TOKEN = '<KEY>'
TWITTER_ACCESS_TOKEN_SECRET = '<KEY>'
TWITTER_AUTH = tweepy.OAuthHandler(TWITTER_CONSUMER_API_KEY, TWITTER_CONSUMER_API_SECRET)
TWITTER_AUTH.set_access_token(TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_TOKEN_SECRET)
TWITTER = tweepy.API(TWITTER_AUTH)
dir(TWITTER)
twitter_user = TWITTER.get_user('elonmusk')
twitter_user
twitter_user.id
elon_tweets = twitter_user.timeline()
elon_tweets
elon_tweets[0].text
len(elon_tweets)
elon_tweets = twitter_user.timeline(count=200,
                                    exclude_replies=True,
                                    include_rts=False,
                                    # max_id='1303802636827799555',
                                    tweet_mode='extended')
len(elon_tweets)
elon_tweets[1].full_text<jupyter_output><empty_output><jupyter_text>SpaCy Embeddings<jupyter_code>import spacy
nlp = spacy.load('en_core_web_md', disable=['tagger', 'parser'])
tokens = nlp("dog cat banana adajksdfhdfg")

for token in tokens:
    print(token.text, token.has_vector, token.is_oov)
dog = nlp('dog')
dog.vector
tweet_embedding = nlp(elon_tweets[1].full_text)
tweet_embedding.vector<jupyter_output><empty_output><jupyter_text>Bringing it all together<jupyter_code>def vectorize_tweet(nlp, tweet_text):
    return nlp(tweet_text).vector
def add_or_update_user(username, nlp):
    try:
        twitter_user = TWITTER.get_user(username)
        tweets = twitter_user.timeline(count=200,
                                    exclude_replies=True,
                                    include_rts=False,
                                    tweet_mode='extended')
        embeddings = vectorize_tweet(nlp, tweets[0].full_text)
    
    except Exception as e:
        print('Error processing {}: {}'.format(username, e))
    
    return tweets, embeddings
    @app.route('/')
    def root():
        return render_template('base.html', title='Home', users=User.query.all())

    @app.route('/user', methods=['POST'])
    @app.route('/user/<name>', methods=['GET'])
    def add_or_update_user(name=None, message=''):
        name = name or request.values['user_name']

        try:
            if request.method == "POST":
                add_user_tweepy(name)
                message = "User {} successfully added!".format(name)
            tweets = User.query.filter(User.username == name).one().tweet
        except Exception as e:
            print(f'Error adding {name}: {e}')
            tweets = []
            
        return render_template('user.html', title=name, tweets=tweets, message=message)

    return app<jupyter_output><empty_output><jupyter_text>Load and Save SpaCy model<jupyter_code>import spacy
import en_core_web_md

#alternative is running: python -m spacy download en_core_web_md from within the heroku shell
nlp = en_core_web_md.load()
nlp.to_disk('../spacy_md_model')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10156472261735419, 'fraction_numerical': 0.011948790896159318, 'mean_word_length': 3.4506329113924052, 'pattern_counts': {'":': 1, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '566678', 'license': 'MIT', 'n_tokens_mistral': 1248, 'n_tokens_neox': 1169, 'n_words': 229, 'path': '03_Unit_3/Sprint3/notebooks/LS332_DSPT7_APIs_Demo.ipynb', 'repo_name': 'mark-morelos/my_DataScience_notes', 'size': '3645'}