<jupyter_start><jupyter_text>Data Could be Interesting on Account of Seasonal Trends? Might not be worth the work of decoding the col names Non_Bus_Travel is good to go and highly informativeI think...<jupyter_code>import pandas as pd
# Data Source
# https://www.fhwa.dot.gov/policyinformation/tables/trafficmonitoring/
aug_laye = pd.read_csv("TMAS2015_august_laye.txt",
                       header=None,
                       names=[
                           'id',
                           'big_neg_int',
                           'lrs_location_point',
                           'big_neg_float',
                           'trailing_float',
                           'state_code',
                           'id2',
                           'one_d',
                           'one_d2',
                           'small_int2',
                           'an_code',
                           'latitude',
                           'longitude',
                           'small_float',
                           'a_code',
                           'small_float2',
                           'small_float3',
                           'zero',
                           'small_int3',
                           'lt1_float',
                           'small_int4',
                           'On NHS'
                       ]
                    )
print(aug_laye.shape)
aug_laye.head()
aug_laye.describe()
aug_laye.info()
aug_laye.iloc[1]
# A is ID since all are unique
aug_laye.iloc[:, 0].nunique() == len(aug_laye.iloc[:, 0])
aug_laye.iloc[:, 1].value_counts().nlargest(5)
# LRS Location Point
# HMMMM Not totally Unique
aug_laye.iloc[:, 2].value_counts().nlargest(5)
# Length is Consistent
for i in aug_laye.iloc[:, 2]:
    assert len(str(i)) == 8
print('All Length of 8.')
aug_laye.iloc[:, 3].value_counts().nlargest(5)
# I think these are state codes
# nunique == 44
aug_laye.iloc[:, 6].value_counts().nlargest(5)
# 32. HPMS Sample Type (Column 140) – Critical
# N = No, not on an HPMS standard sample section
# Y = Yes, on an HPMS standard sample section
# Clearly Yes No

# --- ORRRRRRR ---
# 34. National Highway System (Column 153) – Critical
# N = No, not on National Highway System
# Y = Yes, on National Highway System

aug_laye.iloc[:, 21].value_counts()
# LANESOFCLS: Number of Lanes Monitored for Vehicle Class
# LANESOFVOL: Number of Lanes Monitored for Traffic Volume
# no weight records for an hour might be interpreted to mean that 
# ... the WIM system was not working.
aug_laye.iloc[:, 14].value_counts()
"""
LONGITUDE : Monitoring site longitude
LATITUDE: Monitoring site latitude
STATE: State FIPS
StationID: Traffic Monitoring Site ID. This ID is state unique.
Dir: Direction of Travel
LN: Lane of Travel
YR: Year of Data
FC: Functional Classification
NUMBERLAN: Number of Lanes in Direction Indicated
TYPE: Sample Type for TMAS
LANESOFVOL: Number of Lanes Monitored for Traffic Volume
LANESOFCLS: Number of Lanes Monitored for Vehicle Class
LANESOFWGT: Number of Lanes Monitored for Truck Weight
YR1: Year Station Established
YEARDISC: Year Station Discontinued
COUNTY: FIPS County Code
HPMS: HPMS Sample Type
"""
cardinality = dict()
for i in range(aug_laye.shape[1]):
    card_i = aug_laye.iloc[:, i].nunique()/len(aug_laye.iloc[:, i])
    cardinality.update({i: card_i})
sorted_card = {k: v for k, v in sorted(cardinality.items(), reverse=False, key=lambda item: item[1])}
sorted_card.keys()
aug_laye.iloc[:, 9].value_counts()
aug_laye.iloc[:, 8].value_counts()
# Mostly 0
aug_laye.iloc[:, 19].value_counts()
aug_laye.iloc[:, 17].value_counts()
# 0-8 
# POSTED ROUTE SIGNING CODES?
# Code Includes 1-10
aug_laye.iloc[:, 7].value_counts()
aug_laye.iloc[:, 8].value_counts()
cardinality = dict()
for i in range(aug_laye.shape[1]):
    card_i = aug_laye.iloc[:, i].nunique()/len(aug_laye.iloc[:, i])
    cardinality.update({i: card_i})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1088627049180328, 'fraction_numerical': 0.015368852459016393, 'mean_word_length': 2.2953586497890295, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '133807', 'license': 'MIT', 'n_tokens_mistral': 1312, 'n_tokens_neox': 1254, 'n_words': 351, 'path': 'data/travel_revenue/explore_FHWA.ipynb', 'repo_name': 'taylor-curran/vrb-know', 'size': '3904'}