<jupyter_start><jupyter_text>How to get Long/Lat Data from Tweets-  Data pulled using AppAuth Handler through Tweepy -  Search Query "Climate Change"<jupyter_code>import pandas as pd 
import json
import numpy as np 
%%bash 
pwd
df = pd.read_json("../wyns/data/tweets.txt",  lines=True)  # json file 
#  Check dictionary keys

print (list(df.columns.values))
print ("Corpus contains {0} tweets".format(len(df)))
print (df['place'].dropna())
print (df['place'])
print (df['id'][0])
print (df['full_text'][0])
# Check if Wes' Code catches all tweet text 

full_text = []
i = 0
with open("../wyns/data/tweets.txt") as f:
    for line in f:
        b = json.loads(line)
#         for key in b['retweeted_status'].keys():
#             print(key)
        if 'retweeted_status' in b:
            if 'extended_tweet' in b['retweeted_status']:
              #  print(b['retweeted_status']['extended_tweet']['full_text'])
                full_text.append(b['retweeted_status']['extended_tweet']['full_text'])
                #retweeted status -> usr -> location 
            else:
              #  print(b['text'])
                full_text.append(b['full_text'])
        elif 'extended_tweet' in b:
           # print(b['extended_tweet']['full_text'])
            full_text.append(b['extended_tweet']['full_text'])
        elif 'text' in b:
           # print(b['text'])
            full_text.append(b['full_text'])

        #print(i)
        i += 1
#         break
print('Tweets found: ',len(full_text))
#Long/Lat Format

#print (df['place'][29]['bounding_box']['coordinates'][0][0][1])
#print (df['place'].dropna())
#print (df['place'][8073])
#print (df['geo'].dropna()[8073]['coordinates'][0])
#print (df['truncated'])

print (df['full_text'][64009])
#  Try saving data to 2D array [coordinate, date, full_text]

data = []

with open("../wyns/data/tweets.txt") as f:
    for line in f:
        b = json.loads(line) 
        if b['full_text'] != data:
            if b['geo'] is not None:  # Checks to see if any coord data was saved in geo
                data.append([b['geo']['coordinates'][0],
                             b['geo']['coordinates'][1],
                             b['place']['full_name'],
                             b['created_at'], 
                             b['full_text']])
            elif b['place'] is not None: # Checks to see if any coord data saved in place 
                data.append([b['place']['bounding_box']['coordinates'][0][0][0],
                             b['place']['bounding_box']['coordinates'][0][0][1],
                             b['place']['full_name'],
                             b['created_at'],
                             b['full_text']])
        else:
            pass 

print(np.array(data).shape)
sample = np.array(data)
#print (sample[0:10])
print (sample[0:10])
sample_df = pd.DataFrame(sample, columns=['long','lat','self_reported_location','date','text'])
sample_df.to_csv('../wyns/data/sample.csv', encoding='utf-8', index=False)
sample_df.head()
with open('sample.csv') as f: 
    for line in f:
        print (f)
def check_tweet_data(tweet):
    data = []
    if tweet['geo'] is not None:  # Checks to see if any coord data was saved in geo
        data.append([tweet['geo']['coordinates'][0],
                     tweet['geo']['coordinates'][1],
                     tweet['place']['full_name'],
                     tweet['created_at'], 
                     tweet['text']])
    elif tweet['place'] is not None: # Checks to see if any coord data saved in place 
        data.append([tweet['place']['bounding_box']['coordinates'][0][0][0],
                     tweet['place']['bounding_box']['coordinates'][0][0][1],
                     tweet['place']['full_name'],
                     tweet['created_at'],
                     tweet['text']])
    else:
        pass 
    sample = np.array(data)
    sample_df = pd.DataFrame(sample, columns=['long','lat','self_reported_location','date','text'])
    
def check_if_exists(file_name):
    
    
    with open(file_name, 'a') as f:
    df.to_csv(f, header=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16946676406135866, 'fraction_numerical': 0.01168736303871439, 'mean_word_length': 2.4006622516556293, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '350703', 'license': 'MIT', 'n_tokens_mistral': 1337, 'n_tokens_neox': 1291, 'n_words': 278, 'path': 'examples/Sarah_Tweepy.ipynb', 'repo_name': 'sarahalamdari/DIRECT_capstone', 'size': '4107'}