<jupyter_start><jupyter_text>load packages and modules<jupyter_code>%matplotlib inline
%load_ext autoreload
%autoreload 2
import pandas as pd
import MangroveConservation.get_twitter_data1 as getTwitterdata
import MangroveConservation.clean_text1 as clean
import MangroveConservation.sentiment_analysis1 as sentiment
import MangroveConservation.visualization as visual
%matplotlib inline
help(getTwitterdata.get_data)<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>collect twitter data<jupyter_code>###collect twitter data and save them into CSV
api_key = '<KEY>'
secret_key = '<KEY>'
#DEV_ENVIRONMENT_LABEL = 'mangroveConservation'
#API_SCOPE = 'fullarchive'  # 'fullarchive' for full archive, '30day' for last 31 days

search_query = '-RT mangrove forest'
to_date = '2019-06-19' # format YYYY-MM-DD HH:MM (hour and minutes optional)
from_date = '2019-12-31'  # format YYYY-MM-DD HH:MM (hour and minutes optional)
filename = 'twitter_premium_api_demo1.jsonl'  # Where the Tweets should be saved
csvfile = 'mangrove1.csv'

#getTwitterdata.get_data(search_query,api_key,secret_key,to_date,frome_date,filename)

FILENAME = '/home/gongmimi/CMSE802/MangroveConservation/twitter_premium_api_demo1.jsonl'
csvfile = '/home/gongmimi/CMSE802/MangroveConservation/mangrove1.csv'

tweets = getTwitterdata.load_jsonl(FILENAME)

#getTwitterdata.create_csv(tweets,csvfile)

tweets = clean.import_tweet("/home/gongmimi/CMSE802/MangroveConservation/mangrove1.csv")
#tweets = clean.ImportTweet("/Users/DELL/Dropbox/MangroveConservation/mangrove1.csv")<jupyter_output><empty_output><jupyter_text>exploratory analysis collect the most fewquent words/phrases and graph wordcloud map<jupyter_code>tweets
sentiment.PlotTopWords(tweets["user_description"],22,1,5)
sentiment.PlotTopWords(tweets["tweet"],20,2,4)
sentiment.PlotWordCloud(tweets['user_description'])
sentiment.PlotWordCloud(tweets['tweet'])<jupyter_output><empty_output><jupyter_text>sentiment analysis<jupyter_code>tweets["sentiment"]=sentiment.Sentiment(tweets["tweet"])
tweets.head()
tweets.to_csv("mangrove1_cleaned.csv", index=False, header=True)
sentiment.PlotSentiment(tweets["sentiment"])
negative = tweets[tweets['sentiment']=='Negative']['tweet']
sentiment.PlotTopWords(negative,22,3,5)
positive = tweets[tweets['sentiment']=='Positive']['tweet']
sentiment.PlotTopWords(positive,22,2,4)
neutral = tweets[tweets['sentiment']=='Neutral']['tweet']
sentiment.PlotTopWords(neutral,22,3,5)
neg_user = tweets[tweets['sentiment']=='Negative']['user_description']
sentiment.PlotTopWords(neg_user,22,1,5)
pos_user = tweets[tweets['sentiment']=='Positive']['user_description']
sentiment.PlotTopWords(pos_user,22,1,5)
neu_user = tweets[tweets['sentiment']=='Neutral']['user_description']
sentiment.PlotTopWords(neu_user,22,1,5)<jupyter_output><empty_output><jupyter_text>Data visualization  Visualization<jupyter_code># name of the file with the Tweet objects

overlay,world = clip_polygon(tweets)
overlay,points = centroid_polygon(overlay)
visualization_polygons(overlay,world)<jupyter_output>C:\Users\DELL\Anaconda3\lib\site-packages\ipykernel_launcher.py:29: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
{'fraction_non_alphanumeric': 0.12041884816753927, 'fraction_numerical': 0.02181500872600349, 'mean_word_length': 9.713395638629283, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '557462', 'license': 'MIT', 'n_tokens_mistral': 1336, 'n_tokens_neox': 1210, 'n_words': 195, 'path': 'example.ipynb', 'repo_name': 'lwt852/MangroveConservation', 'size': '3503'}