<jupyter_start><jupyter_text>INTRACITY PREPROCESS<jupyter_code>from datetime import datetime
import pandas as pd
import numpy as np

# classifies the city according to their starting lat,long

def classify_city(s_lat, s_lon):
    if abs(s_lat - 13) <= 1:
        if abs(s_lon - 80.5) <= 1:
            return 0  # delhi
        elif abs(s_lon - 77.5) <= 1:
            return 1  # banglore
    elif abs(s_lon - 72.5) <= 1:
        return 2  # chennai
    elif abs(s_lat - 25.5) <= 1:
        return 3  # lucknow
    elif abs(s_lon - 88.5) <= 1:
        return 4  # bombay
    elif abs(s_lat - 28.5) <= 1:
        return 5  # kolkata
    else:
        return None
    
pass


# if the vehicle is air conditioned 
def isAC(vehicle):

    if vehicle == 1 or vehicle == 2 or vehicle == 4:
        return 1
    else:
        return 0
    
pass

# is the mode of transport is public
def isPublic(vehicle):
    if vehicle == 0 or vehicle == 3 or vehicle == 5:
        return 1
    else:
        return 0


pass


# Returns mean latitude of the city
def findMean_lat(city):
    if city == 0 or city == 1:
        return 13
    elif city == 2:
        return 19.5
    elif city == 3:
        return 25.5
    elif city == 4:
        return 22.5
    elif city == 5:
        return 28.5


pass


# Returns mean longitude of the city
def findMean_long(city):
    if city == 0:
        return 80.5
    elif city == 1:
        return 77.5
    elif city == 2:
        return 72.5
    elif city == 3:
        return 83
    elif city == 4:
        return 88.5
    elif city == 5:
        return 77


pass
def preprocess_city(df, mapping):
    df = df.drop(df[df.STARTING_LATITUDE.isnull() & df.DESTINATION_LATITUDE.isnull()].index)
    
    df.STARTING_LATITUDE.fillna(df.DESTINATION_LATITUDE, inplace=True)
    df.STARTING_LONGITUDE.fillna(df.DESTINATION_LONGITUDE, inplace=True)
    df.DESTINATION_LATITUDE.fillna(df.STARTING_LATITUDE, inplace=True)
    df.DESTINATION_LONGITUDE.fillna(df.STARTING_LONGITUDE, inplace=True)
    df.fillna(-100, inplace=True)
    
    d_format = '%Y-%m-%d %H:%M:%S'
    
    df['TIMESTAMP'] = [datetime.strptime(x, d_format) for x in df['TIMESTAMP']]
    df['MONTH']     = df['TIMESTAMP'].apply(lambda x: x.month)
    df['DAY']       = df['TIMESTAMP'].apply(lambda x: x.day)
    df['YEAR']      = df['TIMESTAMP'].apply(lambda x: x.year)
    df['TIME']      = df['TIMESTAMP'].apply(lambda x: x.strftime("%H%M"))
    df['weekday']   = df['TIMESTAMP'].apply(lambda x: x.dayofweek)
    df['TIME_AM']   = df['TIMESTAMP'].apply(lambda x: 0 if x.strftime("%p") == "AM" else 1)
    df.TIMESTAMP = df['TIMESTAMP'].apply(lambda x: x.timestamp())

    df.VEHICLE_TYPE = df.VEHICLE_TYPE.str.lower()
    df = df.replace({"VEHICLE_TYPE": mapping})
    df['bus'] = df.apply(lambda x: 1
                         if (x['VEHICLE_TYPE'] == 0 or x['VEHICLE_TYPE'] == 4 or x['VEHICLE_TYPE'] == 5)
                         else 0, axis=1)
    df['taxi']      = df.apply(lambda x: 1 if x['VEHICLE_TYPE'] == 6 else 0, axis=1)
    df['cooling']   = df.apply(lambda x: isAC(x['VEHICLE_TYPE']), axis=1)
    df['public']    = df.apply(lambda x: isPublic(x['VEHICLE_TYPE']), axis=1)
    df['city']      = df.apply(lambda x: classify_city(x['STARTING_LATITUDE'], x['STARTING_LONGITUDE']), axis=1)
    df['mean_lat']  = df['city'].apply(lambda x: findMean_lat(x))
    df['mean_long'] = df['city'].apply(lambda x: findMean_long(x))
    
    df.STARTING_LONGITUDE    = df.apply(lambda x: x['mean_long'] - x['STARTING_LONGITUDE'], axis=1)
    df.DESTINATION_LONGITUDE = df.apply(lambda x: x['mean_long'] - x['DESTINATION_LONGITUDE'], axis=1)
    df.STARTING_LATITUDE     = df.apply(lambda x: x['mean_lat'] - x['STARTING_LATITUDE'], axis=1)
    df.DESTINATION_LATITUDE  = df.apply(lambda x: x['mean_lat'] - x['DESTINATION_LATITUDE'], axis=1)
    
    df.reset_index(drop=True)
    return df


pass

# mapping(string --> integer) for VEHICLE_TYPE column
def create_mapping(train_col):
    keys = train_col.str.lower().unique().tolist()
    return dict(zip(keys, range(len(keys))))


pass
# importing data
train = pd.read_csv('../data/intracity_fare_train.csv')
test = pd.read_csv('../data/intracity_fare_test.csv')
id_vec = np.array(test.loc[:,test.columns == 'ID'])
# preprocess the data
vehicle_map = create_mapping(train.VEHICLE_TYPE)
train = preprocess_city(train, vehicle_map)
test = preprocess_city(test, vehicle_map)
# saving the processed data as csv files.
train.to_csv("processed_train.csv", index=False)
test.to_csv("processed_test.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14420131291028446, 'fraction_numerical': 0.023413566739606127, 'mean_word_length': 2.9507346585998273, 'pattern_counts': {'":': 1, '<': 12, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '444086', 'license': 'MIT', 'n_tokens_mistral': 1824, 'n_tokens_neox': 1705, 'n_words': 383, 'path': 'notebooks_&_scripts/.ipynb_checkpoints/Preprocess-checkpoint.ipynb', 'repo_name': 'packetChor/Intracity-Fare-Estimation', 'size': '4570'}