<jupyter_start><jupyter_text>Load json string into tweeta<jupyter_code>import tweeta
json_str = r'{"created_at": "Fri Dec 01 04:53:11 +0000 2017", "id": 936458131114094593, "id_str": "936458131114094593", "text": "@jungcula: Q: alcohol Krystal likes the most: wine\nQ: person she looked for during hard times: family\nQ: nickname she likes th\u2026 ", "source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>", "truncated": false, "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "in_reply_to_screen_name": null, "user": {"id": 152253428, "id_str": "152253428", "name": "\uc138\ud6c8\uc774", "screen_name": "empresshwang", "location": "Ko Ko Bop ", "url": null, "description": "#EXO\n\nEven when its tiring, i will hold onto the 8 pairs of hand\n\n-Chanyeol Park, Hand, 24112017,The ElyXion, Gocheok Sky Dome.", "translator_type": "none", "protected": false, "verified": false, "followers_count": 154, "friends_count": 131, "listed_count": 1, "favourites_count": 4868, "statuses_count": 41886, "created_at": "Sat Jun 05 13:48:12 +0000 2010", "utc_offset": -32400, "time_zone": "Alaska", "geo_enabled": true, "lang": "en", "contributors_enabled": false, "is_translator": false, "profile_background_color": "2C14E0", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/453793495187091456/2WFUp1Sk.png", "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/453793495187091456/2WFUp1Sk.png", "profile_background_tile": true, "profile_link_color": "E80E0E", "profile_sidebar_border_color": "FFFFFF", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": true, "profile_image_url": "http://pbs.twimg.com/profile_images/900384342240776192/9-AmuqtE_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/900384342240776192/9-AmuqtE_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/152253428/1496565300", "default_profile": false, "default_profile_image": false, "following": null, "follow_request_sent": null, "notifications": null}, "geo": null, "coordinates": null, "place": null, "contributors": null, "retweeted_status": {"created_at": "Fri Dec 01 04:18:48 +0000 2017", "id": 936449479254753280, "id_str": "936449479254753280", "text": "Q: alcohol Krystal likes the most: wine\nQ: person she looked for during hard times: family\nQ: nickname she likes th\u2026 https://t.co/C46LzdD0X8", "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>", "truncated": true, "in_reply_to_status_id": 936443673331429377, "in_reply_to_status_id_str": "936443673331429377", "in_reply_to_user_id": 3066798906, "in_reply_to_user_id_str": "3066798906", "in_reply_to_screen_name": "jungcula", "user": {"id": 3066798906, "id_str": "3066798906", "name": "\u3139", "screen_name": "jungcula", "location": "OK", "url": "https://open.spotify.com/user/elizabethynh/playlist/7wzyD6bULRsF31d3EPFgCF", "description": "<NAME> is \ubb54\ub4e4. \ud074\uc4f0\ub9cc \uc0dd\uac01\ub098 #KRYSTALSOLO2K18 -jcl- https://curiouscat.me/jungiddy", "translator_type": "none", "protected": false, "verified": false, "followers_count": 9040, "friends_count": 151, "listed_count": 205, "favourites_count": 682, "statuses_count": 22440, "created_at": "Sat Mar 07 15:47:48 +0000 2015", "utc_offset": 25200, "time_zone": "Jakarta", "geo_enabled": false, "lang": "en", "contributors_enabled": false, "is_translator": false, "profile_background_color": "131516", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/702468469304864768/ajIUe-38.jpg", "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/702468469304864768/ajIUe-38.jpg", "profile_background_tile": true, "profile_link_color": "E81C4F", "profile_sidebar_border_color": "000000", "profile_sidebar_fill_color": "000000", "profile_text_color": "000000", "profile_use_background_image": true, "profile_image_url": "http://pbs.twimg.com/profile_images/921249283835170816/zLjicZBp_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/921249283835170816/zLjicZBp_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/3066798906/1511415698", "default_profile": false, "default_profile_image": false, "following": null, "follow_request_sent": null, "notifications": null}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "extended_tweet": {"full_text": "Q: alcohol Krystal likes the most: wine\nQ: person she looked for during hard times: family\nQ: nickname she likes the most: Ddoodeong-ie\nQ: compliments she likes the most: trustworthy\n\nHAHAHAHA", "display_text_range": [0, 192], "entities": {"hashtags": [], "urls": [], "user_mentions": [], "symbols": []}}, "quote_count": 0, "reply_count": 1, "retweet_count": 61, "favorite_count": 53, "entities": {"hashtags": [], "urls": [{"url": "https://t.co/C46LzdD0X8", "expanded_url": "https://twitter.com/i/web/status/936449479254753280", "display_url": "twitter.com/i/web/status/9\u2026", "indices": [117, 140]}], "user_mentions": [], "symbols": []}, "favorited": false, "retweeted": false, "filter_level": "low", "lang": "en"}, "is_quote_status": false, "quote_count": 0, "reply_count": 0, "retweet_count": 0, "favorite_count": 0, "entities": {"hashtags": [], "urls": [], "user_mentions": [{"screen_name": "jungcula", "name": "\u3139", "id": 3066798906, "id_str": "3066798906", "indices": [3, 12]}], "symbols": []}, "favorited": false, "retweeted": false, "filter_level": "low", "lang": "en", "timestamp_ms": "1512103991665"}'
tweet = tweeta.TweetaTweet(json_str)
json_str = r'{"delete":{"status":{"id":3061373903,"user_id":57136355}}'
tweet = tweeta.TweetaTweet(json_str)<jupyter_output><empty_output><jupyter_text>Get varous versions of the text<jupyter_code>tweet.tweet()
tweet.text()
tweet.fixed_text()<jupyter_output><empty_output><jupyter_text>Get an attribute from tweet<jupyter_code>tweet.get("user")<jupyter_output><empty_output><jupyter_text>Work with `created_at` date formating<jupyter_code>import time
tweet.created_at('%Y-%m')
tweet.created_at('Y-M')<jupyter_output><empty_output><jupyter_text>Play with snowflake (twitter id generator)<jupyter_code>import tweeta.constants
tweeta.constants.PARSE_TIME_FORMAT<jupyter_output><empty_output><jupyter_text>More or less I want to know the relationship between `timestampe` and tweet `id`Example:* id: '151406703034900480'* created_at: 'Mon Dec 26 20:59:05 +0000 2011'<jupyter_code>t = time.strptime('Mon Dec 26 20:59:05 +0000 2011', tweeta.constants.PARSE_TIME_FORMAT)
t
time.mktime(t)<jupyter_output><empty_output><jupyter_text>https://github.com/client9/snowflake2time/blob/master/python/snowflake.py<jupyter_code># Twitter Snowflake ID to timestamp (and back)
# https://github.com/client9/snowflake2time/
# <NAME> @ngalbreath <EMAIL>
# Public Domain -- No Copyright -- Cut-n-Paste!
#  but be kind and give credit
#

# Python by default has unlimited integer precision, so this is mostly
# an exercise in date/time/UTC conversion

import time
import calendar

def str2utc(s):
    # parse twitter time string into UTC seconds, unix-style
    # python's bizarro world of dates, times and calendars
    return calendar.timegm(time.strptime(s, "%a %b %d %H:%M:%S +0000 %Y"))

def utc2snowflake(stamp):
    return (int(round(stamp * 1000)) - 1288834974657) << 22

def snowflake2utc(sf):
    return ((sf >> 22) + 1288834974657) / 1000.0

def str2utcms(s):
    return 1000 * calendar.timegm(time.strptime(s, "%a %b %d %H:%M:%S +0000 %Y"))

def snowflake2utcms(sf):
    return ((sf >> 22) + 1288834974657)

# really is the best way to get utc timestamp?
#   (minus changing your box to be UTC)
def utcnow():
    calendar.timegm(datetime.datetime.utcnow().timetuple())
utc_t = str2utc('Mon Dec 26 20:59:05 +0000 2011')
utc_t
utc2snowflake(utc_t)#151406703034900480<jupyter_output><empty_output><jupyter_text>** Snowflake Layout **1 sign bit -- not used, always 0?41 bits (milliseconds since epoch) - 1288834974657L5 bits datacenter id5 bits machine id12 bit sequence numberSo, to emuerate all possible tweets with the same timestampe, we will have 2^22 = 4,194,304 different combinations.. 151406703034900480 actual id151406700262326272 generated from timestampThe first 8 digits are the same, the rest are different.<jupyter_code>snowflake2utc(151406703034900480)
(1324933145 * 1000 - 1288834974657) << 22
'{0:064b}'.format(1324933145)
len('0000000000000000000000000000000001001110111110001110000000011001')
'{0:064b}'.format((1324933145 * 1000 - 1288834974657) << 22)
len('0000000000000000000000')<jupyter_output><empty_output><jupyter_text>Check whether a tweet is a retweet or quote<jupyter_code>tweet.is_retweet()
tweet.json()
'XIR:%d'%tweet.is_retweet()
tweet.fixed_text()
tweet.get('retweeted_status')
line = '{"created_at": "Fri Dec 01 04:53:05 +0000 2017", "id": 936458105939996674, "id_str": "936458105939996674", "text": "RT @melaniexbarrera: to read \\u201cOU still sucks\\u201d on my tl https://t.co/gZALaBNFsv", "source": "<a href=\\"http://twitter.com/download/iphone\\" rel=\\"nofollow\\">Twitter for iPhone</a>", "truncated": false, "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "in_reply_to_screen_name": null, "user": {"id": 1705999584, "id_str": "1705999584", "name": "Richmas\\ud83c\\udf84(4.0 SZN)", "screen_name": "alexfrommacedon", "location": "Mercedes / ATX", "url": null, "description": "There\'s a hope still burning. John 8:32 The University of Texas \\u201821 \\ud83e\\udd18\\ud83c\\udffb", "translator_type": "none", "protected": false, "verified": false, "followers_count": 781, "friends_count": 540, "listed_count": 8, "favourites_count": 4793, "statuses_count": 28249, "created_at": "Wed Aug 28 01:30:31 +0000 2013", "utc_offset": null, "time_zone": null, "geo_enabled": true, "lang": "en", "contributors_enabled": false, "is_translator": false, "profile_background_color": "C0DEED", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_tile": false, "profile_link_color": "1DA1F2", "profile_sidebar_border_color": "C0DEED", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": true, "profile_image_url": "http://pbs.twimg.com/profile_images/928764655257997314/fsgPfa0p_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/928764655257997314/fsgPfa0p_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1705999584/1512066894", "default_profile": true, "default_profile_image": false, "following": null, "follow_request_sent": null, "notifications": null}, "geo": null, "coordinates": null, "place": null, "contributors": null, "retweeted_status": {"created_at": "Fri Dec 01 04:52:39 +0000 2017", "id": 936457994572845056, "id_str": "936457994572845056", "text": "to read \\u201cOU still sucks\\u201d on my tl https://t.co/gZALaBNFsv", "display_text_range": [0, 33], "source": "<a href=\\"http://twitter.com/download/iphone\\" rel=\\"nofollow\\">Twitter for iPhone</a>", "truncated": false, "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "in_reply_to_screen_name": null, "user": {"id": 769416939571601408, "id_str": "769416939571601408", "name": "\\u24dc \\u24d4 \\u24db", "screen_name": "melaniexbarrera", "location": "listening to depressing music ", "url": "https://vsco.co/voguemela", "description": "big nose but still pulling these hoes.", "translator_type": "none", "protected": false, "verified": false, "followers_count": 410, "friends_count": 295, "listed_count": 0, "favourites_count": 5675, "statuses_count": 1344, "created_at": "Sat Aug 27 06:11:08 +0000 2016", "utc_offset": -28800, "time_zone": "Pacific Time (US & Canada)", "geo_enabled": true, "lang": "en", "contributors_enabled": false, "is_translator": false, "profile_background_color": "F5F8FA", "profile_background_image_url": "", "profile_background_image_url_https": "", "profile_background_tile": false, "profile_link_color": "1DA1F2", "profile_sidebar_border_color": "C0DEED", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": true, "profile_image_url": "http://pbs.twimg.com/profile_images/932408598193418240/fqn4547p_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/932408598193418240/fqn4547p_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/769416939571601408/1509230706", "default_profile": true, "default_profile_image": false, "following": null, "follow_request_sent": null, "notifications": null}, "geo": null, "coordinates": null, "place": null, "contributors": null, "quoted_status_id": 936438127954194433, "quoted_status_id_str": "936438127954194433", "quoted_status": {"created_at": "Fri Dec 01 03:33:42 +0000 2017", "id": 936438127954194433, "id_str": "936438127954194433", "text": "I\\u2019ve been meaning to ask \\ud83e\\udd37\\ud83c\\udffb\\u200d\\u2642\\ufe0f https://t.co/E1wVqaupDM", "display_text_range": [0, 30], "source": "<a href=\\"http://twitter.com/download/iphone\\" rel=\\"nofollow\\">Twitter for iPhone</a>", "truncated": false, "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "in_reply_to_screen_name": null, "user": {"id": 1705999584, "id_str": "1705999584", "name": "Richmas\\ud83c\\udf84(4.0 SZN)", "screen_name": "alexfrommacedon", "location": "Mercedes / ATX", "url": null, "description": "There\'s a hope still burning. John 8:32 The University of Texas \\u201821 \\ud83e\\udd18\\ud83c\\udffb", "translator_type": "none", "protected": false, "verified": false, "followers_count": 781, "friends_count": 540, "listed_count": 8, "favourites_count": 4793, "statuses_count": 28248, "created_at": "Wed Aug 28 01:30:31 +0000 2013", "utc_offset": null, "time_zone": null, "geo_enabled": true, "lang": "en", "contributors_enabled": false, "is_translator": false, "profile_background_color": "C0DEED", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_tile": false, "profile_link_color": "1DA1F2", "profile_sidebar_border_color": "C0DEED", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": true, "profile_image_url": "http://pbs.twimg.com/profile_images/928764655257997314/fsgPfa0p_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/928764655257997314/fsgPfa0p_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1705999584/1512066894", "default_profile": true, "default_profile_image": false, "following": null, "follow_request_sent": null, "notifications": null}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "quote_count": 7, "reply_count": 0, "retweet_count": 0, "favorite_count": 0, "entities": {"hashtags": [], "urls": [], "user_mentions": [], "symbols": [], "media": [{"id": 936438120844640257, "id_str": "936438120844640257", "indices": [31, 54], "media_url": "http://pbs.twimg.com/media/DP7lgmsUQAEJ98p.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7lgmsUQAEJ98p.jpg", "url": "https://t.co/E1wVqaupDM", "display_url": "pic.twitter.com/E1wVqaupDM", "expanded_url": "https://twitter.com/alexfrommacedon/status/936438127954194433/photo/1", "type": "photo", "sizes": {"small": {"w": 576, "h": 518, "resize": "fit"}, "medium": {"w": 576, "h": 518, "resize": "fit"}, "large": {"w": 576, "h": 518, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}}}]}, "extended_entities": {"media": [{"id": 936438120844640257, "id_str": "936438120844640257", "indices": [31, 54], "media_url": "http://pbs.twimg.com/media/DP7lgmsUQAEJ98p.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7lgmsUQAEJ98p.jpg", "url": "https://t.co/E1wVqaupDM", "display_url": "pic.twitter.com/E1wVqaupDM", "expanded_url": "https://twitter.com/alexfrommacedon/status/936438127954194433/photo/1", "type": "photo", "sizes": {"small": {"w": 576, "h": 518, "resize": "fit"}, "medium": {"w": 576, "h": 518, "resize": "fit"}, "large": {"w": 576, "h": 518, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}}}]}, "favorited": false, "retweeted": false, "possibly_sensitive": false, "filter_level": "low", "lang": "en"}, "is_quote_status": true, "quote_count": 1, "reply_count": 0, "retweet_count": 1, "favorite_count": 0, "entities": {"hashtags": [], "urls": [{"url": "https://t.co/gZALaBNFsv", "expanded_url": "https://twitter.com/alexfrommacedon/status/936438127954194433", "display_url": "twitter.com/alexfrommacedo\\u2026", "indices": [34, 57]}], "user_mentions": [], "symbols": []}, "favorited": false, "retweeted": false, "possibly_sensitive": false, "filter_level": "low", "lang": "en"}, "quoted_status_id": 936438127954194433, "quoted_status_id_str": "936438127954194433", "quoted_status": {"created_at": "Fri Dec 01 03:33:42 +0000 2017", "id": 936438127954194433, "id_str": "936438127954194433", "text": "I\\u2019ve been meaning to ask \\ud83e\\udd37\\ud83c\\udffb\\u200d\\u2642\\ufe0f https://t.co/E1wVqaupDM", "display_text_range": [0, 30], "source": "<a href=\\"http://twitter.com/download/iphone\\" rel=\\"nofollow\\">Twitter for iPhone</a>", "truncated": false, "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "in_reply_to_screen_name": null, "user": {"id": 1705999584, "id_str": "1705999584", "name": "Richmas\\ud83c\\udf84(4.0 SZN)", "screen_name": "alexfrommacedon", "location": "Mercedes / ATX", "url": null, "description": "There\'s a hope still burning. John 8:32 The University of Texas \\u201821 \\ud83e\\udd18\\ud83c\\udffb", "translator_type": "none", "protected": false, "verified": false, "followers_count": 781, "friends_count": 540, "listed_count": 8, "favourites_count": 4793, "statuses_count": 28248, "created_at": "Wed Aug 28 01:30:31 +0000 2013", "utc_offset": null, "time_zone": null, "geo_enabled": true, "lang": "en", "contributors_enabled": false, "is_translator": false, "profile_background_color": "C0DEED", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_tile": false, "profile_link_color": "1DA1F2", "profile_sidebar_border_color": "C0DEED", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": true, "profile_image_url": "http://pbs.twimg.com/profile_images/928764655257997314/fsgPfa0p_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/928764655257997314/fsgPfa0p_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1705999584/1512066894", "default_profile": true, "default_profile_image": false, "following": null, "follow_request_sent": null, "notifications": null}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "quote_count": 7, "reply_count": 0, "retweet_count": 0, "favorite_count": 0, "entities": {"hashtags": [], "urls": [], "user_mentions": [], "symbols": [], "media": [{"id": 936438120844640257, "id_str": "936438120844640257", "indices": [31, 54], "media_url": "http://pbs.twimg.com/media/DP7lgmsUQAEJ98p.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7lgmsUQAEJ98p.jpg", "url": "https://t.co/E1wVqaupDM", "display_url": "pic.twitter.com/E1wVqaupDM", "expanded_url": "https://twitter.com/alexfrommacedon/status/936438127954194433/photo/1", "type": "photo", "sizes": {"small": {"w": 576, "h": 518, "resize": "fit"}, "medium": {"w": 576, "h": 518, "resize": "fit"}, "large": {"w": 576, "h": 518, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}}}]}, "extended_entities": {"media": [{"id": 936438120844640257, "id_str": "936438120844640257", "indices": [31, 54], "media_url": "http://pbs.twimg.com/media/DP7lgmsUQAEJ98p.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7lgmsUQAEJ98p.jpg", "url": "https://t.co/E1wVqaupDM", "display_url": "pic.twitter.com/E1wVqaupDM", "expanded_url": "https://twitter.com/alexfrommacedon/status/936438127954194433/photo/1", "type": "photo", "sizes": {"small": {"w": 576, "h": 518, "resize": "fit"}, "medium": {"w": 576, "h": 518, "resize": "fit"}, "large": {"w": 576, "h": 518, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}}}]}, "favorited": false, "retweeted": false, "possibly_sensitive": false, "filter_level": "low", "lang": "en"}, "is_quote_status": true, "quote_count": 0, "reply_count": 0, "retweet_count": 0, "favorite_count": 0, "entities": {"hashtags": [], "urls": [{"url": "https://t.co/gZALaBNFsv", "expanded_url": "https://twitter.com/alexfrommacedon/status/936438127954194433", "display_url": "twitter.com/alexfrommacedo\\u2026", "indices": [55, 78]}], "user_mentions": [{"screen_name": "melaniexbarrera", "name": "\\u24dc \\u24d4 \\u24db", "id": 769416939571601408, "id_str": "769416939571601408", "indices": [3, 19]}], "symbols": []}, "favorited": false, "retweeted": false, "possibly_sensitive": false, "filter_level": "low", "lang": "en", "timestamp_ms": "1512103985663"}'
tweet = tweeta.TweetaTweet(line)
tweet.is_quote()
tweet.get('quoted_status')<jupyter_output><empty_output><jupyter_text>Extract mentions (either from `entities` or `text`)  From text<jupyter_code>from tweeta.text import extract_mentions
tweet._mentions_from_text()
text = tweet.text()
extract_mentions("whatever")
import re
VALID_MENTION_PRECEDING_CHARS = '(?:^|[^a-zA-Z0-9_!#$%&*@＠]|(?:^|[^a-zA-Z0-9_+~.-])(?:rt|RT|rT|Rt):?)'
AT_SIGNS = '[@＠]'

VALID_MENTION  = '('                                       # $0: whole match
VALID_MENTION += '(%s)'%(VALID_MENTION_PRECEDING_CHARS)    # $1: Preceding character
VALID_MENTION += '('                                       # $2: mention
VALID_MENTION += '(%s)'%(AT_SIGNS)                         # $3: At mark
VALID_MENTION += '([a-zA-Z0-9_]{1,20})'                    # $4: Screen name
VALID_MENTION += '(\/[a-zA-Z][a-zA-Z0-9_\-]{0,24})?'       # $5: List (optional)
VALID_MENTION += ')'
VALID_MENTION += ')'

MENTION_RE = re.compile(VALID_MENTION, re.MULTILINE | re.DOTALL | re.UNICODE)
[m[2] for m in MENTION_RE.findall(text)]<jupyter_output><empty_output><jupyter_text>From entities<jupyter_code>tweet.get('entities')
tweet._mentions_from_entities()
tweet.mentions()<jupyter_output><empty_output><jupyter_text>Remove mentions from text<jupyter_code>tweet.mentions()
tweet.text()
tweeta.text.remove_mentions(tweet.text())
tweet.remove_mentions(tweet.text())<jupyter_output><empty_output><jupyter_text>Extract hashtags  From text<jupyter_code>line = r'{"favorited": false, "text": "Stasiun TV yang tetap menggelar perayaan tahun baru besar-besaran dan mewah sungguh keterlaluan!!!! #GusDur wafat", "in_reply_to_user_id": null, "in_reply_to_status_id": null, "in_reply_to_screen_name": null, "geo": null, "source": "<a href=\"http://dabr.co.uk\" rel=\"nofollow\">dabr</a>", "created_at": "Thu Dec 31 00:00:00 +0000 2009", "user": {"profile_sidebar_fill_color": "A0C5C7", "profile_background_tile": false, "url": "http://www.jurpro.net", "profile_sidebar_border_color": "86A4A6", "verified": false, "followers_count": 40, "description": "", "friends_count": 46, "geo_enabled": false, "profile_background_color": "709397", "favourites_count": 1, "created_at": "Sun Sep 20 16:25:19 +0000 2009", "notifications": null, "profile_text_color": "333333", "protected": false, "profile_image_url": "http://a3.twimg.com/profile_images/600299075/Arif_Lukman_normal.jpg", "time_zone": "Jakarta", "location": "Bandung", "name": "<NAME>", "statuses_count": 680, "following": null, "profile_link_color": "FF3300", "screen_name": "ariflukmanbdg", "id": 75819470, "profile_background_image_url": "http://s.twimg.com/a/1262113883/images/themes/theme6/bg.gif", "utc_offset": 25200}, "truncated": false, "id": 7213643202}'
tweet = tweeta.TweetaTweet(line)
from tweeta.text import extract_hashtags
extract_hashtags(tweet.text())
extract_hashtags(" WHATE #HSDF #中文 ")<jupyter_output><empty_output><jupyter_text>From entities<jupyter_code>line = r'{"created_at": "Fri Dec 01 04:53:05 +0000 2017", "id": 936458105931599872, "id_str": "936458105931599872", "text": "RT @Abdulra71165161: \ud83d\udcab\ud83c\udf37\u0644\u062d\u0645\u0627\u064a\u0629 \u0627\u0631\u0636\u064a\u062a\u0643 \u0628\u0634\u0643\u0644 \u0627\u0646\u064a\u0642 \u0648\u0639\u0635\u0631\u064a\n\ud83d\udcab\ud83c\udf37\u0644\u0644\u0627\u0633\u062a\u0641\u0633\u0627\u0631\ud83d\udc47\ud83c\udffb\n\ud83d\udcf10501374440\n #\u0645\u0648\u0627\u0644\u064a\u062f_\u062f\u064a\u0633\u0645\u0628\u0631 https://t.co/rsUj7yupLs", "source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>", "truncated": false, "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "in_reply_to_screen_name": null, "user": {"id": 3435244280, "id_str": "3435244280", "name": "\u0634\u0639\u0627\u0639 \u0627\u0644\u0646\u0648\u0631 \u2075\u2075\u1d37", "screen_name": "mngom95", "location": "\u0627\u0644\u0631\u064a\u0627\u0636", "url": null, "description": "#\u0645\u0648\u0627\u0641\u0642\u0629_\u062a\u0628\u0627\u062f\u0644 #\u062b\u0642\u0629_\u0645\u0646_\u0645\u0641\u0636\u0644\u062a\u0649 \ud83c\udf20\u062a\u0628\u0627\u062f\u0644 \ud83c\udf1f5/5 \ud83c\udf1f  \ud83c\udf1f\u0644\u0627 \u062a\u0633\u062a\u0623\u0630\u0646 \u0623\u062f\u062e\u0644 \u0645\u0641\u0636\u0644\u062a\u064a \u0647\u0627\u062a 5 \u0648\u0639\u0637\u0646\u064a \u062a\u0645 \u0648\u0644\u0643 \u0628\u0627\u0644\u0645\u062b\u0644\ud83c\udf1f", "translator_type": "none", "protected": false, "verified": false, "followers_count": 59928, "friends_count": 56654, "listed_count": 308, "favourites_count": 869, "statuses_count": 67749, "created_at": "Thu Sep 03 09:04:46 +0000 2015", "utc_offset": -28800, "time_zone": "Pacific Time (US & Canada)", "geo_enabled": false, "lang": "ar", "contributors_enabled": false, "is_translator": false, "profile_background_color": "C0DEED", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "profile_background_tile": false, "profile_link_color": "1DA1F2", "profile_sidebar_border_color": "C0DEED", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": true, "profile_image_url": "http://pbs.twimg.com/profile_images/920674989501583361/aQOaHTDZ_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/920674989501583361/aQOaHTDZ_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/3435244280/1508341229", "default_profile": true, "default_profile_image": false, "following": null, "follow_request_sent": null, "notifications": null}, "geo": null, "coordinates": null, "place": null, "contributors": null, "retweeted_status": {"created_at": "Fri Dec 01 03:58:41 +0000 2017", "id": 936444413966012416, "id_str": "936444413966012416", "text": "\ud83d\udcab\ud83c\udf37\u0644\u062d\u0645\u0627\u064a\u0629 \u0627\u0631\u0636\u064a\u062a\u0643 \u0628\u0634\u0643\u0644 \u0627\u0646\u064a\u0642 \u0648\u0639\u0635\u0631\u064a\n\ud83d\udcab\ud83c\udf37\u0644\u0644\u0627\u0633\u062a\u0641\u0633\u0627\u0631\ud83d\udc47\ud83c\udffb\n\ud83d\udcf10501374440\n #\u0645\u0648\u0627\u0644\u064a\u062f_\u062f\u064a\u0633\u0645\u0628\u0631 https://t.co/rsUj7yupLs", "display_text_range": [0, 73], "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>", "truncated": false, "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "in_reply_to_screen_name": null, "user": {"id": 845285819795591168, "id_str": "845285819795591168", "name": "Abdulrahman123", "screen_name": "Abdulra71165161", "location": "\u0627\u0644\u0645\u0645\u0644\u0643\u0629 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \u0627\u0644\u0633\u0639\u0648\u062f\u064a\u0629", "url": null, "description": "....\u200f\u200f\u062f\u0644\u0639 \u0645\u0648\u062a\u0631\u0643.....  \n\u0627\u0636\u0641 \u0644\u0645\u0633\u0647 \u062c\u0645\u0627\u0644\u064a\u0647 \u0645\u0645\u064a\u0632\u0647 \u0644\u062f\u062e\u0644\u064a\u0629 \u0633\u064a\u0627\u0631\u062a\u0643  \n\u062f\u0639\u0633\u0627\u062a \u0623\u0631\u0636\u064a\u0647 \u062c\u0644\u062f \u0641\u0627\u062e\u0631  \u062a\u0641\u0635\u064a\u0644  \u0644\u0633\u064a\u0627\u0631\u0627\u0643\n\u0644\u062a\u0648\u0627\u0635\u0644 0501374440", "translator_type": "none", "protected": false, "verified": false, "followers_count": 3286, "friends_count": 4990, "listed_count": 61, "favourites_count": 111, "statuses_count": 961, "created_at": "Fri Mar 24 14:46:58 +0000 2017", "utc_offset": null, "time_zone": null, "geo_enabled": false, "lang": "ar", "contributors_enabled": false, "is_translator": false, "profile_background_color": "F5F8FA", "profile_background_image_url": "", "profile_background_image_url_https": "", "profile_background_tile": false, "profile_link_color": "1DA1F2", "profile_sidebar_border_color": "C0DEED", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": true, "profile_image_url": "http://pbs.twimg.com/profile_images/859110692548431873/8fKX6NxQ_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/859110692548431873/8fKX6NxQ_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/845285819795591168/1506812755", "default_profile": true, "default_profile_image": false, "following": null, "follow_request_sent": null, "notifications": null}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "quote_count": 0, "reply_count": 0, "retweet_count": 18, "favorite_count": 2, "entities": {"hashtags": [{"text": "\u0645\u0648\u0627\u0644\u064a\u062f_\u062f\u064a\u0633\u0645\u0628\u0631", "indices": [59, 73]}], "urls": [], "user_mentions": [], "symbols": [], "media": [{"id": 936444374141100032, "id_str": "936444374141100032", "indices": [74, 97], "media_url": "http://pbs.twimg.com/media/DP7rMmCX4AA1_S8.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7rMmCX4AA1_S8.jpg", "url": "https://t.co/rsUj7yupLs", "display_url": "pic.twitter.com/rsUj7yupLs", "expanded_url": "https://twitter.com/Abdulra71165161/status/936444413966012416/photo/1", "type": "photo", "sizes": {"medium": {"w": 900, "h": 1200, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "large": {"w": 960, "h": 1280, "resize": "fit"}, "small": {"w": 510, "h": 680, "resize": "fit"}}}]}, "extended_entities": {"media": [{"id": 936444374141100032, "id_str": "936444374141100032", "indices": [74, 97], "media_url": "http://pbs.twimg.com/media/DP7rMmCX4AA1_S8.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7rMmCX4AA1_S8.jpg", "url": "https://t.co/rsUj7yupLs", "display_url": "pic.twitter.com/rsUj7yupLs", "expanded_url": "https://twitter.com/Abdulra71165161/status/936444413966012416/photo/1", "type": "photo", "sizes": {"medium": {"w": 900, "h": 1200, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "large": {"w": 960, "h": 1280, "resize": "fit"}, "small": {"w": 510, "h": 680, "resize": "fit"}}}, {"id": 936444374136840194, "id_str": "936444374136840194", "indices": [74, 97], "media_url": "http://pbs.twimg.com/media/DP7rMmBW4AIyg7W.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7rMmBW4AIyg7W.jpg", "url": "https://t.co/rsUj7yupLs", "display_url": "pic.twitter.com/rsUj7yupLs", "expanded_url": "https://twitter.com/Abdulra71165161/status/936444413966012416/photo/1", "type": "photo", "sizes": {"large": {"w": 960, "h": 1280, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "small": {"w": 510, "h": 680, "resize": "fit"}, "medium": {"w": 900, "h": 1200, "resize": "fit"}}}, {"id": 936444374195625984, "id_str": "936444374195625984", "indices": [74, 97], "media_url": "http://pbs.twimg.com/media/DP7rMmPX4AABaL5.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7rMmPX4AABaL5.jpg", "url": "https://t.co/rsUj7yupLs", "display_url": "pic.twitter.com/rsUj7yupLs", "expanded_url": "https://twitter.com/Abdulra71165161/status/936444413966012416/photo/1", "type": "photo", "sizes": {"medium": {"w": 900, "h": 1200, "resize": "fit"}, "small": {"w": 510, "h": 680, "resize": "fit"}, "large": {"w": 960, "h": 1280, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}}}, {"id": 936444374183043074, "id_str": "936444374183043074", "indices": [74, 97], "media_url": "http://pbs.twimg.com/media/DP7rMmMX4AI4ns3.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7rMmMX4AI4ns3.jpg", "url": "https://t.co/rsUj7yupLs", "display_url": "pic.twitter.com/rsUj7yupLs", "expanded_url": "https://twitter.com/Abdulra71165161/status/936444413966012416/photo/1", "type": "photo", "sizes": {"medium": {"w": 900, "h": 1200, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "large": {"w": 960, "h": 1280, "resize": "fit"}, "small": {"w": 510, "h": 680, "resize": "fit"}}}]}, "favorited": false, "retweeted": false, "possibly_sensitive": false, "filter_level": "low", "lang": "ar"}, "is_quote_status": false, "quote_count": 0, "reply_count": 0, "retweet_count": 0, "favorite_count": 0, "entities": {"hashtags": [{"text": "\u0645\u0648\u0627\u0644\u064a\u062f_\u062f\u064a\u0633\u0645\u0628\u0631", "indices": [80, 94]}], "urls": [], "user_mentions": [{"screen_name": "Abdulra71165161", "name": "Abdulrahman123", "id": 845285819795591168, "id_str": "845285819795591168", "indices": [3, 19]}], "symbols": [], "media": [{"id": 936444374141100032, "id_str": "936444374141100032", "indices": [95, 118], "media_url": "http://pbs.twimg.com/media/DP7rMmCX4AA1_S8.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7rMmCX4AA1_S8.jpg", "url": "https://t.co/rsUj7yupLs", "display_url": "pic.twitter.com/rsUj7yupLs", "expanded_url": "https://twitter.com/Abdulra71165161/status/936444413966012416/photo/1", "type": "photo", "sizes": {"medium": {"w": 900, "h": 1200, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "large": {"w": 960, "h": 1280, "resize": "fit"}, "small": {"w": 510, "h": 680, "resize": "fit"}}, "source_status_id": 936444413966012416, "source_status_id_str": "936444413966012416", "source_user_id": 845285819795591168, "source_user_id_str": "845285819795591168"}]}, "extended_entities": {"media": [{"id": 936444374141100032, "id_str": "936444374141100032", "indices": [95, 118], "media_url": "http://pbs.twimg.com/media/DP7rMmCX4AA1_S8.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7rMmCX4AA1_S8.jpg", "url": "https://t.co/rsUj7yupLs", "display_url": "pic.twitter.com/rsUj7yupLs", "expanded_url": "https://twitter.com/Abdulra71165161/status/936444413966012416/photo/1", "type": "photo", "sizes": {"medium": {"w": 900, "h": 1200, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "large": {"w": 960, "h": 1280, "resize": "fit"}, "small": {"w": 510, "h": 680, "resize": "fit"}}, "source_status_id": 936444413966012416, "source_status_id_str": "936444413966012416", "source_user_id": 845285819795591168, "source_user_id_str": "845285819795591168"}, {"id": 936444374136840194, "id_str": "936444374136840194", "indices": [95, 118], "media_url": "http://pbs.twimg.com/media/DP7rMmBW4AIyg7W.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7rMmBW4AIyg7W.jpg", "url": "https://t.co/rsUj7yupLs", "display_url": "pic.twitter.com/rsUj7yupLs", "expanded_url": "https://twitter.com/Abdulra71165161/status/936444413966012416/photo/1", "type": "photo", "sizes": {"large": {"w": 960, "h": 1280, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "small": {"w": 510, "h": 680, "resize": "fit"}, "medium": {"w": 900, "h": 1200, "resize": "fit"}}, "source_status_id": 936444413966012416, "source_status_id_str": "936444413966012416", "source_user_id": 845285819795591168, "source_user_id_str": "845285819795591168"}, {"id": 936444374195625984, "id_str": "936444374195625984", "indices": [95, 118], "media_url": "http://pbs.twimg.com/media/DP7rMmPX4AABaL5.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7rMmPX4AABaL5.jpg", "url": "https://t.co/rsUj7yupLs", "display_url": "pic.twitter.com/rsUj7yupLs", "expanded_url": "https://twitter.com/Abdulra71165161/status/936444413966012416/photo/1", "type": "photo", "sizes": {"medium": {"w": 900, "h": 1200, "resize": "fit"}, "small": {"w": 510, "h": 680, "resize": "fit"}, "large": {"w": 960, "h": 1280, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}}, "source_status_id": 936444413966012416, "source_status_id_str": "936444413966012416", "source_user_id": 845285819795591168, "source_user_id_str": "845285819795591168"}, {"id": 936444374183043074, "id_str": "936444374183043074", "indices": [95, 118], "media_url": "http://pbs.twimg.com/media/DP7rMmMX4AI4ns3.jpg", "media_url_https": "https://pbs.twimg.com/media/DP7rMmMX4AI4ns3.jpg", "url": "https://t.co/rsUj7yupLs", "display_url": "pic.twitter.com/rsUj7yupLs", "expanded_url": "https://twitter.com/Abdulra71165161/status/936444413966012416/photo/1", "type": "photo", "sizes": {"medium": {"w": 900, "h": 1200, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "large": {"w": 960, "h": 1280, "resize": "fit"}, "small": {"w": 510, "h": 680, "resize": "fit"}}, "source_status_id": 936444413966012416, "source_status_id_str": "936444413966012416", "source_user_id": 845285819795591168, "source_user_id_str": "845285819795591168"}]}, "favorited": false, "retweeted": false, "possibly_sensitive": false, "filter_level": "low", "lang": "ar", "timestamp_ms": "1512103985661"}'
tweet = tweeta.TweetaTweet(line)
tweet.get('entities')
tweet.hashtags()<jupyter_output><empty_output><jupyter_text>Langugage detection<jupyter_code>tweet.is_en()
tweet.get('lang')
tweet.text()
from tweeta.text import lang
lang(tweet.text())
lang(" WHATE #HSDF #中文 ")
tweet.get('user')
del tweet._tweet['user']['lang']
tweet.is_user_en()<jupyter_output><empty_output><jupyter_text>Text cleanning<jupyter_code>from tweeta.text import sanitize_nofunccall
sanitize_nofunccall("whatever me/you\n whatever\n")<jupyter_output><empty_output><jupyter_text>is_geotagged<jupyter_code>tweet.is_geotagged()<jupyter_output><empty_output><jupyter_text>is_deleted()<jupyter_code>import tweeta

line = r'{"delete":{"status":{"id":3061373903,"user_id":57136355}}}'
tweet = tweeta.TweetaTweet(line)
tweet.is_deleted()<jupyter_output><empty_output><jupyter_text>user name/screen_name<jupyter_code>tweet.user_screen_name()<jupyter_output><empty_output><jupyter_text>user description<jupyter_code>tweeta.text.sanitize_nofunccall(tweet.user_description())
tweet.user_description()<jupyter_output><empty_output><jupyter_text>id/id_str<jupyter_code>builtins.reload(tweeta)
tweet.tweet_id()
tweet.user_id()<jupyter_output><empty_output><jupyter_text>json<jupyter_code>tweet.json()<jupyter_output><empty_output><jupyter_text>urls<jupyter_code>import re
FLAGS = re.MULTILINE | re.DOTALL | re.UNICODE

URL_RE = re.compile(r"https?:\/\/\S+\b|www\.(\w+\.)+\S*", FLAGS)
url_text = " whatever https://hello.co "
URL_RE.sub("", url_text)
url_text
m = URL_RE.search(url_text)
m
tweet._raw_text = url_text
tweet.has_url()
retweet_ = tweet.get('retweeted_status')
type(retweet_)
retweet = tweeta.TweetaTweet(retweet_)
retweet.text()<jupyter_output><empty_output><jupyter_text>is_valid()<jupyter_code>line = r'{"user": {"profile_use_background_image": true, "notifications": null, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/638720895205384192/gzcmCbFB.jpg", "description": "baby girl gets wtv baby girl wants | champagnemami |sc : tatianaoliv |", "default_profile_image": false, "utc_offset": 3600, "listed_count": 7, "is_translator": false, "favourites_count": 28046, "created_at": "Wed May 01 15:46:51 +0000 2013", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1394933906/1484181190", "statuses_count": 52467, "contributors_enabled": false, "location": "2615", "friends_count": 970, "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/638720895205384192/gzcmCbFB.jpg", "profile_background_color": "FFFFFF", "id_str": "1394933906", "time_zone": "Amsterdam", "verified": false, "profile_text_color": "DCDCDA", "geo_enabled": true, "profile_image_url": "http://pbs.twimg.com/profile_images/821442411855577089/lhnBByy4_normal.jpg", "default_profile": false, "profile_image_url_https": "https://pbs.twimg.com/profile_images/821442411855577089/lhnBByy4_normal.jpg", "profile_sidebar_border_color": "000000", "profile_sidebar_fill_color": "F7F7F7", "protected": false, "following": null, "follow_request_sent": null, "url": null, "name": "crystal", "id": 1394933906, "lang": "pt", "screen_name": "tt_oliv", "profile_link_color": "FF691F", "followers_count": 2577, "profile_background_tile": true}, "geo": null, "retweet_count": 0, "in_reply_to_status_id": null, "in_reply_to_user_id_str": null, "is_quote_status": false, "created_at": "Fri Jan 20 00:14:57 +0000 2017", "retweeted": false, "filter_level": "low", "entities": {"symbols": [], "user_mentions": [], "urls": [], "hashtags": []}, "contributors": null, "in_reply_to_user_id": null, "favorited": false, "timestamp_ms": "1484871297665", "id_str": "822235933739286528", "favorite_count": 0, "coordinates": null, "in_reply_to_screen_name": null, "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>", "truncated": false, "place": null, "text": "t cheia d dores", "id": 822235933739286528, "lang": "pt", "in_reply_to_status_id_str": null}'
line = r'{"created_at": "Sat Jan 07 22:11:12 +0000 2017", "id": 817856136615436288, "id_str": "817856136615436288", "full_text": "RT @Levijacob10: @Levijacob10 @esthe_fhany \n1,3,4,5,7", "truncated": false, "display_text_range": [0, 53], "entities": {"hashtags": [], "symbols": [], "user_mentions": [{"screen_name": "Levijacob10", "name": "<NAME> \u26ab\ud83d\udd34", "id": 4585555932, "id_str": "4585555932", "indices": [3, 15]}, {"screen_name": "Levijacob10", "name": "<NAME> \u26ab\ud83d\udd34", "id": 4585555932, "id_str": "4585555932", "indices": [17, 29]}, {"screen_name": "esthe_fhany", "name": "\ud83d\udeae", "id": 737711223802859521, "id_str": "737711223802859521", "indices": [30, 42]}], "urls": []}, "source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>", "in_reply_to_status_id": null, "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_user_id_str": null, "in_reply_to_screen_name": null, "user": {"id": 737711223802859521, "id_str": "737711223802859521", "name": "\ud83d\udeae", "screen_name": "esthe_fhany", "location": "", "description": "30/10\u2764\ufe0f t\u00e9cnica de enfermagem \ud83d\udc69\ud83c\udffd\u200d\u2695\ufe0f\u2764\ufe0f", "url": null, "entities": {"description": {"urls": []}}, "protected": false, "followers_count": 696, "friends_count": 437, "listed_count": 1, "created_at": "Tue May 31 18:23:56 +0000 2016", "favourites_count": 6332, "utc_offset": null, "time_zone": null, "geo_enabled": true, "verified": false, "statuses_count": 7491, "lang": "pt", "contributors_enabled": false, "is_translator": false, "is_translation_enabled": false, "profile_background_color": "F5F8FA", "profile_background_image_url": null, "profile_background_image_url_https": null, "profile_background_tile": false, "profile_image_url": "http://pbs.twimg.com/profile_images/945772757391937537/kQ-Q0a-j_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/945772757391937537/kQ-Q0a-j_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/737711223802859521/1512044907", "profile_link_color": "1DA1F2", "profile_sidebar_border_color": "C0DEED", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": true, "has_extended_profile": false, "default_profile": true, "default_profile_image": false, "following": null, "follow_request_sent": null, "notifications": null, "translator_type": "none"}, "geo": null, "coordinates": null, "place": null, "contributors": null, "retweeted_status": {"created_at": "Sat Jan 07 22:03:00 +0000 2017", "id": 817854071482843136, "id_str": "817854071482843136", "full_text": "@Levijacob10 @esthe_fhany \n1,3,4,5,7", "truncated": false, "display_text_range": [13, 36], "entities": {"hashtags": [], "symbols": [], "user_mentions": [{"screen_name": "Levijacob10", "name": "<NAME> \u26ab\ud83d\udd34", "id": 4585555932, "id_str": "4585555932", "indices": [0, 12]}, {"screen_name": "esthe_fhany", "name": "\ud83d\udeae", "id": 737711223802859521, "id_str": "737711223802859521", "indices": [13, 25]}], "urls": []}, "source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>", "in_reply_to_status_id": 808833253604020224, "in_reply_to_status_id_str": "808833253604020224", "in_reply_to_user_id": 4585555932, "in_reply_to_user_id_str": "4585555932", "in_reply_to_screen_name": "Levijacob10", "user": {"id": 4585555932, "id_str": "4585555932", "name": "<NAME> \u26ab\ud83d\udd34", "screen_name": "Levijacob10", "location": "Rio de Janeiro, Brasil", "description": "", "url": null, "entities": {"description": {"urls": []}}, "protected": false, "followers_count": 859, "friends_count": 310, "listed_count": 2, "created_at": "Thu Dec 17 15:44:08 +0000 2015", "favourites_count": 13390, "utc_offset": null, "time_zone": null, "geo_enabled": true, "verified": false, "statuses_count": 12736, "lang": "pt", "contributors_enabled": false, "is_translator": false, "is_translation_enabled": false, "profile_background_color": "F5F8FA", "profile_background_image_url": null, "profile_background_image_url_https": null, "profile_background_tile": false, "profile_image_url": "http://pbs.twimg.com/profile_images/945817669911097344/hKsJb1Gg_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/945817669911097344/hKsJb1Gg_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/4585555932/1514742132", "profile_link_color": "1DA1F2", "profile_sidebar_border_color": "C0DEED", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "profile_use_background_image": true, "has_extended_profile": true, "default_profile": true, "default_profile_image": false, "following": null, "follow_request_sent": null, "notifications": null, "translator_type": "none"}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "retweet_count": 1, "favorite_count": 1, "favorited": false, "retweeted": false, "lang": "und"}, "is_quote_status": false, "retweet_count": 1, "favorite_count": 0, "favorited": false, "retweeted": false, "lang": "und"}'
line = r'{"text": "@piolilla DA!!!!!!!!!!", "truncated": false, "source": "<a href=\"http://www.twhirl.org/\" rel=\"nofollow\">twhirl</a>", "in_reply_to_status_id": 4510825220, "favorited": false, "created_at": "Thu Oct 01 00:00:26 +0000 2009", "user": {"time_zone": "Santiago", "favourites_count": 1, "profile_background_color": "9ae4e8", "location": "Santiago", "verified": false, "notifications": null, "profile_background_image_url": "http://s.twimg.com/a/1254344155/images/themes/theme1/bg.png", "profile_text_color": "000000", "description": "Soy chilena, soy periodista, me gusta la fotografia, las peliculas, andar en bici, el cafe y el sol.", "following": null, "profile_link_color": "0000ff", "url": null, "profile_sidebar_fill_color": "e0ff92", "protected": false, "created_at": "Fri Apr 04 02:19:35 +0000 2008", "followers_count": 102, "profile_image_url": "http://a3.twimg.com/profile_images/377574333/eos_normal.jpg", "name": "<NAME>", "profile_sidebar_border_color": "87bc44", "id": 14299021, "statuses_count": 785, "profile_background_tile": false, "utc_offset": -14400, "friends_count": 88, "screen_name": "ximepoblete"}, "in_reply_to_user_id": 10925762, "id": 4511052908, "in_reply_to_screen_name": "piolilla"}'
import tweeta
tweeta.__version__
tweet = tweeta.TweetaTweet(line)
tweet
tweet.is_valid()
tweet.text()
import tweeta
with open("test", "r") as f:
    for line in f:
        tweet = tweeta.TweetaTweet(line)
        print(tweet.fixed_text())
        print(tweet.replace_emoji_in_text())
with open("test", "r") as f:
    for line in f:
        tweet = tweeta.TweetaTweet(line)
        print(tweet.fixed_text())
        print(tweet.replace_emoji_in_text("%%"))<jupyter_output>@kotuzak @BetteMidler Oh...ok. conservative, Gun loving trumptard.😅😅😅😅😅😅😅😅 PORN STARS...$130,000 #NRABloodMoney #RussianSanctions  #draftdodger
@kotuzak @BetteMidler Oh...ok. conservative, Gun loving trumptard.%%%%%%%%%%%%%%%% PORN STARS...$130,000 #NRABloodMoney #RussianSanctions  #draftdodger
{'fraction_non_alphanumeric': 0.19825043436098427, 'fraction_numerical': 0.13695502848599944, 'mean_word_length': 7.79825808745112, 'pattern_counts': {'":': 1326, '<': 137, '<?xml version=': 0, '>': 135, 'https://': 90, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '90593', 'license': 'MIT', 'n_tokens_mistral': 24069, 'n_tokens_neox': 19750, 'n_words': 3451, 'path': 'tweeta_test.ipynb', 'repo_name': 'bianjiang/tweeta', 'size': '49551'}