<jupyter_start><jupyter_text>* * * Testando *folium* e *data frames*<jupyter_code>tweets = pd.read_json("data/small-data-fixed.json")
print 'OK!'
type(tweets)
tweets.info()
coordinate = []
for col in tweets['coordinates'][~tweets['coordinates'].isnull()]:
    coord = col['coordinates'][::-1]
    coordinate.append(coord)
    
print coordinate[10]
coord_text = []
for col in tweets['text'][~tweets['coordinates'].isnull()]:
    coord = col.encode('utf-8')
    coord_text.append(coord)
    
print coord_text[10]
tweets[['coordinates','text']][~tweets['coordinates'].isnull()].head(11)
coords = tweets['coordinates']
coords = coords[~coords.isnull()]
coords = coords.apply(lambda d: d['coordinates'][::-1])
coords.head(20)
m = folium.Map([-14,-53.25], zoom_start=4)

for x, text in enumerate(coord_text):
    folium.Marker(coordinate[x], popup=str(coordinate[x])).add_to(m)

m
tweets.text.head()
tweets.user.head()
df = pd.DataFrame()
df['text'] = tweets['text']
df['coordinates'] = tweets['coordinates']
df['user'] = tweets['user']
df.head()


# df['text'] = map(lambda df: df['text'].encode('utf-8'), tweets)
# df['user'] = map(lambda df: df['user']['screen_name'], tweets)<jupyter_output><empty_output><jupyter_text>* * *  Funções de teste<jupyter_code>def datetimeify(df):
    df['created_at'] = pd.DatetimeIndex(df.created_at)
    return df
def sentiment(df):
    text = df.dropna(subset=['text']).text
    sentiment = text.apply(lambda text: TextBlob(text).sentiment)
    df['polarity'] = sentiment.apply(lambda sentiment: sentiment.polarity)
    df['subjectivity'] = sentiment.apply(lambda sentiment: sentiment.subjectivity)
    return df
def influence(df):
    internal = np.sqrt(df.user_followers_count.apply(lambda x: x + 1))
    external = np.sqrt(df.retweet_count.apply(lambda x: x + 1))
    df['influence'] = internal * external
    return df
def influenced_polarity(df):
    df['influenced_polarity'] = df.polarity * df['influence']
    return df
def georeference(df):
    def place_to_coordinate(place_str, kind):
        if pd.isnull(place_str):
            return float('nan')
        number_matcher = r'(-?\d+\.\d+)[,\]]'
        coordinates = re.findall(number_matcher, place_str)
        coordinate = tuple(float(n) for n in coordinates[:2])

        if kind == 'longitude':
            return coordinate[0]
        elif kind == 'latitude':
            return coordinate[1]
    df['latitude'] = df.place.apply(place_to_coordinate, kind='latitude')
    df['longitude'] = df.place.apply(place_to_coordinate, kind='longitude')

    return df
def preprocess(df):
    return (df.pipe(datetimeify))
def preprocess_df(df):
    cleaned = df.pipe(set_hashtags)
    copy = cleaned.copy()
    return preprocess(copy)

def load_df(input_filename):
    raw_df = pd.read_json(input_filename)
    return preprocess(raw_df)

print 'OK'<jupyter_output><empty_output><jupyter_text>* * * Horários de Tweets<jupyter_code>tweets['created_at'] = pd.to_datetime(pd.Series(tweets['created_at']))

tweets.set_index('created_at', drop=False, inplace=True)

tweets.index = tweets.index.tz_localize('GMT')
tweets.index = tweets.index - DateOffset(hours = 3)
tweets.index

tweets.head()
tweets30s = tweets['created_at'].resample('1h', how='count')
tweets30s.head()
avg = tweets30s.mean()

vincent.core.initialize_notebook()
area = vincent.Area(tweets30s)
area.colors(brew='Spectral')
area.display()<jupyter_output><empty_output><jupyter_text>* * * Testes sentimental analysis<jupyter_code>emoticons_str = r"""
    (?:
        [:=;] # Eyes
        [oO\-]? # Nose (optional)
        [D\)\]\(\]/\\OpP] # Mouth
    )"""
 
regex_str = [
    emoticons_str,
    r'<[^>]+>', # HTML tags
    r'(?:@[\w_]+)', # @-mentions
    r"(?:\#+[\w_]+[\w\'_\-]*[\w_]+)", # hash-tags
    r'http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-f][0-9a-f]))+', # URLs
 
    r'(?:(?:\d+,?)+(?:\.?\d+)?)', # numbers
    r"(?:[a-z][a-z'\-_]+[a-z])", # words with - and '
    r'(?:[\w_]+)', # other words
    r'(?:\S)' # anything else
]
    
tokens_re = re.compile(r'('+'|'.join(regex_str)+')', re.VERBOSE | re.IGNORECASE)
emoticon_re = re.compile(r'^'+emoticons_str+'$', re.VERBOSE | re.IGNORECASE)
 
def tokenize(s):
    return tokens_re.findall(s)
 
def preprocess(s, lowercase=True):
    tokens = tokenize(s)
    if lowercase:
        tokens = [token if emoticon_re.search(token) else token.lower() for token in tokens]
    return tokens
 
tweet = "RT @medeirosthiiago: testando exemplo TCC! :D http://example.com #ImpeachmentDay"
print(preprocess(tweet))
# ['RT', '@marcobonzanini', ':', 'just', 'an', 'example', '!', ':D', 'http://example.com', '#NLP']<jupyter_output>['rt', '@medeirosthiiago', ':', 'testando', 'exemplo', 'tcc', '!', ':D', 'http://example.com', '#impeachmentday']
{'fraction_non_alphanumeric': 0.20332561565986107, 'fraction_numerical': 0.008419280151547043, 'mean_word_length': 4.532013969732247, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '78152', 'license': 'MIT', 'n_tokens_mistral': 1849, 'n_tokens_neox': 1800, 'n_words': 296, 'path': 'src/twitter/notebooks/test-notebook.ipynb', 'repo_name': 'medeirosthiiago/glue-grade', 'size': '4751'}