<jupyter_start><jupyter_text><jupyter_code>nltk.download('all')
import nltk
text = '''
A trained language model generates text. We can optionally pass it some text as input, which influences its output. 
The output is generated from what the model “learned” during its training period where it scanned vast amounts of text. 
Training is the process of exposing the model to lots of text. That process has been completed. All the experiments you see now 
are from that one trained model. It was estimated to cost 355 GPU years and cost $4.6m. The dataset of 300 billion tokens of text 
is used to generate training examples for the model. For example, these are three training examples generated from the one sentence at the top.
You can see how you can slide a window across all the text and make lots of examples.
'''

for text1 in text:
  sentences = nltk.sent_tokenize(text1)
  for sentence in sentences:
    words = nltk.word_tokenize(sentence)
    tagged = nltk.pos_tag(words)
    print(tagged)
from nltk.tokenize import TweetTokenizer
text = '''Wind and solar are the cheapest source of new power for two-thirds of the world's population.'''
twtk = TweetTokenizer()
twtk.tokenize(text)
from urllib import request
url = "http://www.gutenberg.org/files/2557/2557-0.txt"
resp = request.urlopen(url)
raw = resp.read().decode('utf8')
len(raw)
print(raw[:75])
from nltk.tokenize import word_tokenize
tokens = word_tokenize(raw)
print(tokens)
#A Python script to download all the tweets of a hashtag into a csv

import tweepy
import csv
import pandas as pd
####input your credentials here
consumer_key = ''
consumer_secret = ''
access_token = ''
access_token_secret = ''

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth,wait_on_rate_limit=True)
#####United Airlines
# Open/Create a file to append data
csvFile = open('ua.csv', 'a')
#Use csv Writer
csvWriter = csv.writer(csvFile)

for tweet in tweepy.Cursor(api.search,q="#unitedAIRLINES",count=100,
                           lang="en",
                           since="2017-04-03").items():
    print (tweet.created_at, tweet.text)
    csvWriter.writerow([tweet.created_at, tweet.text.encode('utf-8')])
import nltk
f = open('tweets1.txt', 'r')
text = f.read()
text1 = text.split()
text2 = nltk.Text(text1)
text2.concordance('good')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09700249791840133, 'fraction_numerical': 0.01623646960865945, 'mean_word_length': 4.498855835240275, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '237206', 'license': 'MIT', 'n_tokens_mistral': 787, 'n_tokens_neox': 742, 'n_words': 267, 'path': 'NLP_week3.ipynb', 'repo_name': 'holps-7/NLP-weekly', 'size': '2402'}