<jupyter_start><jupyter_text>TODO:- Clean (see other notebook)- Create "distance from Seattle" feature<jupyter_code>import pandas as pd
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import scipy.stats as st
# import pymc3 as pm
import seaborn as sns

# enables inline plots, without it plots don't show up in the notebook
%matplotlib inline
%config InlineBackend.figure_format = 'svg'
# %config InlineBackend.figure_format = 'png'
# mpl.rcParams['figure.dpi']= 300
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 25)
pd.set_option('display.precision', 3)
pd.set_option('display.float_format', lambda x: '%.3f' % x)
import pickle
def load_pd_pkl(file):
    import pandas as pd
    try:
        with open(f"{file}.pkl",'rb') as picklefile:
            return pickle.load(picklefile)
    except FileNotFoundError:
        df = pd.read_csv(f"{file}.csv")
        with open(f"{file}.pkl", 'wb') as picklefile:
            pickle.dump(df, picklefile)
        return df
df = load_pd_pkl('data/raw_wta_df')
#df.to_csv('data/raw_wta_df.csv')
df.shape
df.reset_index(drop=True, inplace=True)
#rename columns
df['votes'] = df['votes'].astype(int)
df['countreports'] = df['countreports'].astype(int)
df['rating'] = df['rating'].astype(float)
df['length'] = df['length'].astype(float)
df['gain'] = df['gain'].astype(float)
df['hpoint'] = df['hpoint'].astype(float)
df['lat'] = df['lat'].astype(float)
df['long'] = df['long'].astype(float)
df.info()
# Clean hikes which faultily have the 'hpoint' assigned to the "length" to NaN
df.loc[df['length'] == df['hpoint'], 'length'] = np.nan
# Good! All length types exist with a length preceding it.
df[df['length'].notna() & df['length'].isna()]
# Accurately update length type of "of" to "miles_of_trails" like on website
df[df.lengthtype == "of"]
df.loc[df['lengthtype'] == "of", 'lengthtype'] = "miles_of_trails"

# Clean hikes which faultily have the 'gain assigned to the "length" to NaN
df.loc[df['length'] == df['gain'], 'length'] = np.nan
df[df.length == df.gain].shape

# rename author1 and author2
df.rename(columns={'author1': 'org_author', "author2":"author"},inplace=True)

# do some renaming of dfs and create new one dropping a few columns and all NaNs
old_df = df.copy()
df = df.drop(columns=['trailhead1', 'trailhead2','org_author', 'author', 'subregion','lat','long'])
df.dropna(inplace=True)

# drop hikes without any votes and ratings remaining
df = df.loc[df['votes'] != 0, :]
# There are 7 passes, and also none and n/a
df.fee.unique()
##### consider limitations on extreme values in length and gain at this point
##### should I require votes to be above a certain threshold? 1-5?<jupyter_output><empty_output><jupyter_text>Assumptions and changes to data on first pass:* Must have 3 votes are greater* Remove "one-way" hikes* Remove extreme continuous variables (length, high-point, elevation gain)<jupyter_code>Run below (commented out) tweaks to make tweaks
# df = df[df.votes > 3] # Remove hikes with 3 or less votes contributing to rating
# df = df[df.length > 30] # Remove hikes greater than 30 miles
# df = df[df.gain > 6000] # Remove hikes greater than 6000 feet in vertical gain
# df = df[df.lengthtype != "one-way"] # Remove one way hikes<jupyter_output><empty_output><jupyter_text>To-Do* Feature engineer lat/long* Change and/or remove one-way and miles of trails locations.<jupyter_code># import pickle

# with open('data/cleaned_wta_df.pkl', 'wb') as picklefile:
#     pickle.dump(df, picklefile)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1455927907631653, 'fraction_numerical': 0.009293156857223317, 'mean_word_length': 5.264550264550264, 'pattern_counts': {'":': 1, '<': 13, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '496921', 'license': 'MIT', 'n_tokens_mistral': 1245, 'n_tokens_neox': 1224, 'n_words': 356, 'path': 'wta_data_cleaning.ipynb', 'repo_name': 'spencertollefson/WA-trails-rating-prediction', 'size': '3551'}