<jupyter_start><jupyter_text>This Code Transforms text data (tweets) into hourly waits for the Edmonds-Kingston Washington State Ferry run  Import necessary packages<jupyter_code>import pandas as pd
import glob
import numpy as np<jupyter_output><empty_output><jupyter_text>load data<jupyter_code>all_files = glob.glob("./data/*.csv")

df_from_each_file = (pd.read_csv(f) for f in all_files)
df = pd.concat(df_from_each_file, ignore_index=True)
df.columns
df.columns = df.columns.str.lower().str.replace(' ', '_')<jupyter_output><empty_output><jupyter_text>Extract wait times from tweets Assumptions:* wait-related tweets contain the word 'wait'* edmonds-related tweets contain the full word 'edmonds'* kingston-related tweets contain the full word 'kingston'* numbers are spelled out or numerals Standard tweet Examples:* Edm/King - Edmonds and Kingston Terminal Status - 2 Hour Wait* Edm/King - Kingston Terminal Status - Two Hour Wait* Edm/King - Edmonds Terminal Status - One Hour Wait* Edm/King - No Extended Wait for Drivers Departing Edmonds* Edm/King - One Hour Wait Departing Kingston and Edmonds Non-Standard tweet Examples:* Edm/King - Update - Edmonds and Kingston Terminal Status, 2hrs Edm, 1hr King* Edm/King - No Extended Wait in Kingston - One Hour Wait in Edmonds, Late Vessel* Edm/King - Kingston 6:25am Departure is Cancelled. One Hr. Wait* Edm/King - no longer an extended wait departing edmonds or kingston* Edm/King - edmonds and kingston terminal status - 2 hour wait<jupyter_code># changing text to lowercase and removing the url, 
df['tweet_text'] = df['tweet_text'].str.lower().str.replace('https:.*', '')

# removing route indicator ('edm/king -'), extra whitespace
df['tweet_text'] = df['tweet_text'].str.replace('edm/king -', '').str.strip()

# removing wsp boarding pass indicataor
df['tweet_text'] = df['tweet_text'].str.replace(', no wsp boarding pass required|, wsp boarding pass required', '')
for tweet in df['tweet_text'][0:10]:
    print(tweet)
df_wait = df[df['tweet_text'].str.contains('wait')]
ed_df = df_wait[df_wait['tweet_text'].str.contains('edmonds')].copy()
ki_df = df_wait[df_wait['tweet_text'].str.contains('kingston')].copy()
df_wait.head()
def get_hours(texts, locName, otherLocName, altNames):
    """Returns the hours of wait described in the texts for the location name
    
    Args:
      texts: a panadas series of text describing the waits
      locName: the name of the location to extract the time for
      otherLocName: the name of the location NOT getting times extracted
      altNames: a dictionary of the short names for locations (eg. {'Edmonds': ['Edm', 'Edms', 'E', 'Ed'],
                                                                    'Kingston': ['K', 'Kgstn', 'King']})
      
    Returns: list of ints representing the wait hours in each record of text for the specified location
    """
    
    return [get_hour(text, locName, otherLocName, altNames) for text in texts]
    
import re

def get_hour(text, locName, otherLocName, altNames):
    """Returns the hour of wait described in the text for the location name
    
    Args:
      text: a string describing the wait
      locName: the name of the location to extract the time for
      otherLocName: the name of the location NOT getting times extracted
      altNames: a dictionary of the short names for locations (eg. {'Edmonds': ['Edm', 'Edms', 'E', 'Ed'],
                                                                    'Kingston': ['K', 'Kgstn', 'King']})
      
    Returns: and int - the number of hours wait for the locName
    """
    # initialize hour and backup hour
    hour = None
    backup_hour = None
    
    # check for locName
    if locName not in text:
        return hour
    
    # check for otherLocName (ie. a dual-location message), and get the number for solo messages
    if otherLocName not in text:
        hour = get_num(text)
        
    # otherwise it's a dual-location message
    else:
        # split the message on - and ,
        split_texts = re.split('-|,', text)
        for split_text in split_texts:
            # initialize flag of this section containing the location of interest
            loc_in_text = False
            
            # check for full name and alternate/abreviated names
            if locName in split_text:
                loc_in_text = True
            for name in altNames[locName]:
                if name in split_text:
                    loc_in_text = True
            
            # if the location is in the split, get the hour
            if loc_in_text:
                hour = get_num(split_text)
            
            # if the location isn't in the split, get the hour as a backup
            else:
                backup_hour = get_num(split_text)
                
        # if none of the sections have the name and hour together, use the backup_hour
        if hour == None:
            hour = backup_hour
    return hour
        
import re

def get_num(text):
    """Returns the number contained in the text (assumes only one number 
    between 1 and 3, text or numeric). Returns 0 if text includes no, 
    extenced, and wait."""
    if bool(re.search('1|one|60 minute', text)): return 1
    elif bool(re.search('2|two', text)): return 2
    elif bool(re.search('3|three', text)): return 3
    elif bool(re.search('4|four', text)): return 4
    elif bool(re.search('90 min', text)): return 1.5
    elif bool(re.search('no.*wait', text)): return 0
    else: return None<jupyter_output><empty_output><jupyter_text>Investigating results - uncategorized tweets<jupyter_code>altNames = {'edmonds': ['edm', 'edms', ' ed'], 'kingston': ['kgstn', 'king']}
ed_df['hours'] = get_hours(ed_df['tweet_text'], 'edmonds', 'kingston', altNames)
for tweet in ed_df.loc[ed_df['hours'].isna(),'tweet_text']:
    print(tweet)
ki_df['hours'] = get_hours(ki_df['tweet_text'], 'kingston', 'edmonds', altNames)
ki_df[ki_df['hours'].isna()]
for tweet in ki_df.loc[ki_df['hours'].isna(),'tweet_text']:
    print(tweet)
ed_df.head(20)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10542814717043392, 'fraction_numerical': 0.0049496782709123905, 'mean_word_length': 3.060281312793034, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '116483', 'license': 'MIT', 'n_tokens_mistral': 1872, 'n_tokens_neox': 1800, 'n_words': 632, 'path': 'blog_code.ipynb', 'repo_name': 'allisonhonold/tweets_to_data_ferry_wait_blog', 'size': '6061'}