<jupyter_start><jupyter_text>Prepare Data: load, clean and save<jupyter_code>import pandas as pd
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
import matplotlib.dates as mdates

from datetime import datetime as dt
import datetime

import code.get_mta_data as gmd
import code.process_mta_data as pmd<jupyter_output><empty_output><jupyter_text>import data<jupyter_code>mta = gmd.load_local_data([2019, 2018], [3, 4, 5])
# mta = gmd.load_local_data([2019], [3])<jupyter_output><empty_output><jupyter_text>Tidy up data<jupyter_code>mta = pmd.clean_col_names(mta)
mta = pmd.add_datetime(mta)
mta = pmd.convert_date_to_datetime(mta)<jupyter_output><empty_output><jupyter_text>There are many rows with 'RECOVER AUD'. In Lara's presentation, these were shown to be be duplicates in at least some cases. We will remove them all.<jupyter_code>mta = pmd.remove_recovr_aud(mta)<jupyter_output><empty_output><jupyter_text>[disabled] Select only the entries where time is midnight<jupyter_code># mta = pmd.filter_to_midnight(mta)
mta
filename = 'ekand_clean_data_0.2.csv'
gmd.save_interem_ekand_v_0_1(mta, filename)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09523809523809523, 'fraction_numerical': 0.017316017316017316, 'mean_word_length': 8.248, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '522921', 'license': 'MIT', 'n_tokens_mistral': 461, 'n_tokens_neox': 440, 'n_words': 97, 'path': 'notebooks/submission_2_ekand_make_clean_data_0.2-Copy1.ipynb', 'repo_name': 'ekand/mtacular', 'size': '1155'}