<jupyter_start><jupyter_text>Get all tweets from 511 Central VA as of 13:40pm Oct 14<jupyter_code>js = api.user_timeline('511centralva')
len(js)
while True:
    temp = api.user_timeline('511centralva', count=200, max_id=js[-1]._json['id'])
    if js[-1]._json['id'] == temp[-1]._json['id']:
        break
    else:
        js += temp
len(js), js[-1]._json['id']
fname = 'data_' + at()+ '.pkl'
with open(fname , 'wb') as f:
    # Pickle the 'data' dictionary using the highest protocol available.
    pickle.dump(js, f, pickle.HIGHEST_PROTOCOL)
fname
fname
## reading back

with open('data_201710141356.pkl', 'rb') as f:
    # The protocol version used is detected automatically, so we do not
    # have to specify it.
    older = pickle.load(f)
len(older)
js_dict = {
    'id': [_.id for _ in older],
    'screen_name': [ _.user.screen_name for _ in older],
    'created_at': [_.created_at for _ in older],
    'text': [_.text for _ in older]
}
olddf = pd.DataFrame(js_dict)
olddf.head()
js_dict = {
    'id': [_.id for _ in js],
    'screen_name': [ _.user.screen_name for _ in js],
    'created_at': [_.created_at for _ in js],
    'text': [_.text for _ in js]
}
data = pd.DataFrame(js_dict)
data.head()
#plot = plt.figure(figsize=(400,800))
data.text.map(lambda x: x.split(' ')[0]).value_counts().plot(kind='bar')
plt.show()
data.text.map(lambda x: x.split(':')[0]).value_counts()
data.to_excel('tweets_next.xlsx')
data[data.text.str.startswith(':')]
len(olddf.append(data))
dummy = pd.concat([data, olddf], ignore_index=True)
dummy = dummy.drop_duplicates('id')
len(dummy), dummy.index.has_duplicates
dummy.text.iloc[1082] = 'Incident' + dummy.iloc[1082].text
dummy.iloc[1082].text
dummy['col_cnt'] = dummy.text.map(lambda x:len(x.split(':')))
dummy.col_cnt.value_counts()
dummy[dummy.col_cnt == 3].head()
dummy[dummy.col_cnt == 4].head()
dummy[dummy.col_cnt == 3]['text'].map(lambda x: x.split(':')[0]).value_counts()
dummy[dummy.col_cnt == 4]['text'].map(lambda x: x.split(':')[0]).value_counts()
dummy[dummy.col_cnt == 5]['text'].map(lambda x: x.split(':')[0]).value_counts()
dummy[dummy.col_cnt == 5]['text']
texts = dummy[dummy.text.str.contains('brush')]['text']
import re
pattern = re.compile(r'((?P<status>\w+): )?'
                     r'(?P<advisory>(Advisory|Closed): )?'
                     r'(?P<type>(\w*\s?)+): '
                     r'(?P<direction>[NEWS]B )?on '
                     r'(?P<hwy>.*)( at)? '
                     r'((?P<loc>.*)) in '
                     r'(?P<city>[A-Za-z0-9 ]+).'
                     r'(?P<comment>[a-zA-Z0-9\.&;/ ]+.)?'
                     r'(?P<time>\d+:\d+[AP]M)$'
                    )
attrs = ['status', 'type', 'direction', 'hwy', 'loc', 'city', 'comment', 'time']
for a in attrs:
    t = 'Update: Accident: EB on I-64 at MM199 in Henrico Co. All travel lanes closed. Delay 1 mi.5:52PM'
    t = 'Cleared: Accident: WB on I-64 at MM187 in Richmond.6:24PM'
    print(f'{a} -> {pattern.match(t).group(a)}')
dummy.iloc[1520]
dummy.drop(1520, inplace=True)
dummy.drop(1519, inplace=True)
dummy.iloc[1519]
dummy.iloc[1518]
dummy.drop(1518, inplace=True)
dummy.drop(1527, inplace=True)
dummy.drop(1528, inplace=True)
dummy[dummy.text.str.contains('Diascund')]
for idx in dummy.index:
    if idx%500 == 0:
        print(idx)
    try:
        m = pattern.match(dummy.loc[idx,'text'])
        for a in attrs:
            dummy.loc[idx,a] = m.group(a)
    except AttributeError:
        #if 'Water' not in t and 'opening' not in t:
        print (dummy.loc[idx,'text'])
        #print(out)
    #print ('-'*40)
import seaborn as sns
#plt.style.use('seaborn-talk')
plt.style.use('fivethirtyeight')
dummy.tail()
dummy.status.hasnans, dummy.type.hasnans
for idx in dummy[dummy.type.isnull()].index:
    dummy.drop(idx, inplace=True)
reports = dummy[dummy.status.isnull()]
len(reports)
reports.created_at.dt.date.value_counts().plot()
plt.title('Daily incident reported since Sep 21, 2017')
plt.ylabel('Reported Count')
plt.xlabel('Date')
plt.show()
reports.type.value_counts(dropna=False)[:10].plot(kind='bar')
plt.ylabel('Reported Count')
plt.title('Top 10 types of traffic related events in Central VA')
plt.show()
reports.city.value_counts(dropna=False)[:10].plot(kind='bar')
plt.ylabel('Reported Count')
plt.title('Top 10 areas for traffic related events in Central VA')
plt.show()
temp = reports.groupby(['type', 'city'])['id'].count().sort_values(ascending=False)[:20].to_frame()
temp.head()
def hitype(x):
    return (x in ['Accident', 'Incident', 'Disabled Vehicle'])
#reports.groupby(['type', 'city'])['id'].count().sort_values(ascending=False)[:20].plot.bar(stacked=True)
#temp.unstack('type').plot.bar()# order=.value_counts().iloc[:10].index)
sns.countplot(data=reports[reports.type.map(hitype)], x='city', hue='type', order=reports.city.value_counts().iloc[:5].index)
plt.title('Top traffic incidents by area')
plt.show()
reports.groupby(['type', 'city'])['id'].count().plot.bar()
#.type.value_counts(dropna=False)[:10]
plt.ylabel('Reported Count')
plt.title('Top 10 types of traffic related incidents in the area')
plt.show()
dummy.city.value_counts(dropna=False)[:10].plot(kind='bar')
plt.show()
def hitype(x):
    return (x in ['Accident', 'Incident', 'Disabled Vehicle'])

def hiloc(x):
    return x in ('MM74','MM78','MM76','MM75','MM9','MM79','MM185','MM13','MM18','MM2','MM61','MM3','MM58','MM62','MM0','MM84','MM38','MM186')

#reports.groupby(['type', 'city'])['id'].count().sort_values(ascending=False)[:20].plot.bar(stacked=True)
#temp.unstack('type').plot.bar()# order=.value_counts().iloc[:10].index)
sns.countplot(data=reports[reports.type.map(hitype) & reports['loc'].map(hiloc)], x='hwy', hue='loc', order=reports.city.value_counts().iloc[:5].index)
plt.title('Top traffic incidents by area')
plt.show()
reports[reports.type == 'Accident'].groupby(['hwy','loc'])['id'].count().sort_values(ascending=False)[:10].plot(kind='bar')
plt.title('Accident prone areas')
plt.xlabel('Highway, Milemarker')
plt.ylabel('Accident Count')
plt.show()
type(reports)
reports.head()
reports['dt'] = pd.to_datetime(reports.time)
reports['itime'] = reports.dt.hour()
reports['dt'].dt.time()
reports['hour'] = reports['dt'].dt.hour
reports.head()
#acc = reports[reports.type == 'Accident'].groupby('hour')['type'].to_frame()
#acc
sns.countplot(data=reports[reports.type == 'Accident'], x='hour', palette='Reds_d')
plt.title('Accident frequency through the day')
plt.show()
for t in dummy['text'][1525:1530]:
    print(t)
i = 1528
#pattern.match(dummy['text'][i]), 
dummy['text'][i]
dummy[dummy.text.str.contains('aintenance')]
dummy.drop(1528, inplace=True)
pattern1 = re.compile(r'((?P<status>\w+): )?'
                     r'(?P<advisory>(Advisory|Closed): )?'
                     r'(?P<type>(\w*\s?)+): '
                     r'(?P<direction>.*)? [oO]n '
                     r'(?P<hwy>.*)( at)? '
                     r'((?P<loc>.*)) in '
                     r'(?P<city>[A-Za-z0-9 ]+).'
                     r'(?P<comment>[a-zA-Z0-9\.&;/ ]+.)'
                     r'(?P<time>\d+:\d+[AP]M)$'
                    )
pattern1.match(dummy['text'][i])
from functools import partial
def extract(pos, txt):
    try:
        if 'at ' in txt:
            return re.match(patgat,txt).groups()[pos] 
        else:
            return re.match(patg,txt).groups()[pos]
    except AttributeError:
        return '-'
extract0 = partial(extract, 0)
extract1 = partial(extract, 1)
extract2 = partial(extract, 2)
extract3 = partial(extract, 3)
extract4 = partial(extract, 4)
extract5 = partial(extract, 5)
#dummy['status'] = ''
#dummy[dummy.col_cnt == 4]['status'] = 
dummy[dummy.col_cnt == 4]['text'].map(extract0)
dummy.tail()<jupyter_output><empty_output><jupyter_text>status:<jupyter_code>x = 'Update: Accident: SB on I-95 at MM71 in Richmond. No lanes closed. Delay 2 mi.6:18PM'
#x = 'Update: Accident: SB on VA-288 at MM14 in Chesterfield Co. 1 travel lane closed. Delay 2 mi.6:32PM'
#x = 'Update: Advisory: Other Weather: NB on Rt. 604N at MM2 in Lancaster Co. All NB &amp; all SB travel lanes closed.1:52PM'
x = 'Cleared: Accident: NB on US-17 (George Washington Memorial Hwy) in Gloucester Co.1:30PM'
import re
pat = r'\w+: \w+: .*\.\d+:\d+[AP]M'
patgat = r'(\w+): (\w+): (\w+) on (.*) at (.*) in ([A-Za-z ]+\.)(.*)(\d+:\d+[AP]M)'
patg = r'(\w+): (\w+): (\w+) on (.*) in ([A-Za-z ]+\.)(.*)(\d+:\d+[AP]M)'
pattern = r'^((\w+): )+(\w+) on (.*) (at (.*))? in ([A-Za-z ]+\.)(.*)(\d+:\d+[AP]M)$'
o = re.match(patg, x)
re.match(pattern, dummy.text.iloc[3246]).groups()
dummy.text.iloc[3247]
for _ in dummy[dummy.text.str.contains(r'Standing Water')]['text']:
    print(_)
dummy[dummy.text.str.contains('bridge ')]
sample = js[-1]
sample._json
from rply import LexerGenerator
lg = LexerGenerator()
lg.add(tokebs
      `
      )
lexer = lg.build()
iterator = lexer.kex()`
iterator.next()
api.user_timeline?
import numpy as np

from bokeh.plotting import figure, output_notebook, show
from bokeh.models import LogColorMapper, LogTicker, ColorBar

#output_file('color_bar.html')
output_notebook()

def normal2d(X, Y, sigx=1.0, sigy=1.0, mux=0.0, muy=0.0):
    z = (X-mux)**2 / sigx**2 + (Y-muy)**2 / sigy**2
    return np.exp(-z/2) / (2 * np.pi * sigx * sigy)

X, Y = np.mgrid[-3:3:100j, -2:2:100j]
Z = normal2d(X, Y, 0.1, 0.2, 1.0, 1.0) + 0.1*normal2d(X, Y, 1.0, 1.0)
image = Z * 1e6

color_mapper = LogColorMapper(palette="Viridis256", low=1, high=1e7)

plot = figure(x_range=(0,1), y_range=(0,1), toolbar_location=None)
plot.image(image=[image], color_mapper=color_mapper,
           dh=[1.0], dw=[1.0], x=[0], y=[0])

color_bar = ColorBar(color_mapper=color_mapper, ticker=LogTicker(),
                     label_standoff=12, border_line_color=None, location=(0,0))

plot.add_layout(color_bar, 'right')

show(plot)
from rply import LexerGenerator
lg = LexerGenerator()
#x = 'Update: Accident: SB on I-95 at MM71 in Richmond. No lanes closed. Delay 2 mi.6:18PM'
#x = 'Update: Accident: SB on VA-288 at MM14 in Chesterfield Co. 1 travel lane closed. Delay 2 mi.6:32PM'
#x = 'Update: Advisory: Other Weather: NB on Rt. 604N at MM2 in Lancaster Co. All NB &amp; all SB travel lanes closed.1:52PM'
#x = 'Cleared: Accident: NB on US-17 (George Washington Memorial Hwy) in Gloucester Co.1:30PM'
lg.ignore(r'\s+')
lg.add("Update", "Update:")
lg.add("Accident", "Accident:")
lg.add("Cleared", "Cleared:")
lg.add("Advisory", "Advisory:")
lg.add("on", "on")
lg.add("in", "in")
lg.add("city")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20946915351506457, 'fraction_numerical': 0.034337637494022, 'mean_word_length': 4.100487804878049, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '324564', 'license': 'MIT', 'n_tokens_mistral': 4322, 'n_tokens_neox': 3974, 'n_words': 810, 'path': 'twitter.ipynb', 'repo_name': 'atulsaurav/central_va_traffic_analysis', 'size': '10455'}