<jupyter_start><jupyter_text>Descubrimientos de patrones de accidentes de tránsito en la CDMX  Creación del Dataset Final para generar el modelo de clasificación<jupyter_code>from math import radians, cos, sin, asin, sqrt, atan2
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mpl
from pylab import rcParams
import seaborn as sns
import folium
import networkx as nx
import osmnx as ox
import geopandas as gpd
from shapely.geometry import Point, Polygon, LineString, MultiLineString
import shapefile as shp
import warnings
warnings.filterwarnings('ignore')
pd.set_option('display.max_columns', None)
sns.set(context='paper', style='ticks', palette='inferno')
sns.mpl.rc("figure", figsize=(25, 21))
mpl.rcParams['figure.dpi']= 200
ind_feat50  = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1//results_static//csv_ind_50m.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
ind_feat100 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1//results_static//csv_ind_100m.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
ind_feat200 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1//results_static//csv_ind_200m.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
ind_feat300 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1//results_static//csv_ind_300m.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
ind_feat500 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1//results_static//csv_ind_500m.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)


ind_feat50.info()
train_dir_feat50  = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/results_dynamic/geo_50_train_data.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
train_dir_feat100 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/results_dynamic/geo_100_train_data.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
train_dir_feat200 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/results_dynamic/geo_200_train_data.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
train_dir_feat300 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/results_dynamic/geo_300_train_data.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
train_dir_feat500 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/results_dynamic/geo_500_train_data.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)

test_dir_feat50  = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/results_dynamic/geo_50_test_data.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
test_dir_feat100 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/results_dynamic/geo_100_test_data.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
test_dir_feat200 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/results_dynamic/geo_200_test_data.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
test_dir_feat300 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/results_dynamic/geo_300_test_data.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)
test_dir_feat500 = pd.read_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/results_dynamic/geo_500_test_data.csv',sep = ',', encoding = 'utf-8',header = 0, index_col =False)<jupyter_output><empty_output><jupyter_text>Generamos nuestra variable objetivo<jupyter_code>def create_dep_var(row):
    if (row['PER_ATRAPADO_DESBARRANCADO'] + 
        row['VEH_ATRAPADO'] + 
        row['VEH_DESBARRANCADO'] + 
        row['VOLCADURA'] + 
        row['ACC_FATAL'] + 
        row['ATROPELLADO_FATAL']+
        row['ATROPELLADO']+
        row['ACC_C_LESION']+
        row['FERROVIARIO']+
        row['MOTOCICLISTA']+
        row['OTROS']+
        row['ACC_PRENSADO']+
        row['ACC_S_LESION']) == 0:
        return 0
    else:
        return 1
train_dir_feat50['SINIESTRO']  = train_dir_feat50.apply(create_dep_var, axis=1)
test_dir_feat50['SINIESTRO']   =  test_dir_feat50.apply(create_dep_var, axis=1)

train_dir_feat100['SINIESTRO'] = train_dir_feat100.apply(create_dep_var, axis=1)
test_dir_feat100['SINIESTRO']  =  test_dir_feat100.apply(create_dep_var, axis=1)

train_dir_feat200['SINIESTRO'] = train_dir_feat200.apply(create_dep_var, axis=1)
test_dir_feat200['SINIESTRO']  =  test_dir_feat200.apply(create_dep_var, axis=1)

train_dir_feat300['SINIESTRO'] = train_dir_feat300.apply(create_dep_var, axis=1)
test_dir_feat300['SINIESTRO']  =  test_dir_feat300.apply(create_dep_var, axis=1)

train_dir_feat500['SINIESTRO'] = train_dir_feat500.apply(create_dep_var, axis=1)
test_dir_feat500['SINIESTRO']  =  test_dir_feat500.apply(create_dep_var, axis=1)
train_dir_feat50 = train_dir_feat50[[ 'ACC_C_LESION_100',
                                      'ACC_C_LESION_200',
                                      'ACC_FATAL_100',
                                      'ACC_FATAL_200',
                                      'ACC_PRENSADO_100',
                                      'ACC_PRENSADO_200',
                                      'ACC_S_LESION_100',
                                      'ACC_S_LESION_200',
                                      'ATROPELLADO_100',
                                      'ATROPELLADO_200',
                                      'ATROPELLADO_FATAL_100',
                                      'ATROPELLADO_FATAL_200',
                                      'FERROVIARIO_100',
                                      'FERROVIARIO_200',
                                      'MOTOCICLISTA_100',
                                      'MOTOCICLISTA_200',
                                      'OTROS_100',
                                      'OTROS_200',
                                      'PER_ATRAPADO_DESBARRANCADO_100',
                                      'PER_ATRAPADO_DESBARRANCADO_200',
                                      'VEH_ATRAPADO_100',
                                      'VEH_ATRAPADO_200',
                                      'VEH_DESBARRANCADO_100',
                                      'VEH_DESBARRANCADO_200',
                                      'VOLCADURA_100',
                                      'VOLCADURA_200',
                                      'SINIESTRO']]

train_dir_feat100 = train_dir_feat100[['ACC_C_LESION_100',
                                       'ACC_C_LESION_200',
                                       'ACC_FATAL_100',
                                       'ACC_FATAL_200',
                                       'ACC_PRENSADO_100',
                                       'ACC_PRENSADO_200',
                                       'ACC_S_LESION_100',
                                       'ACC_S_LESION_200',
                                       'ATROPELLADO_100',
                                       'ATROPELLADO_200',
                                       'ATROPELLADO_FATAL_100',
                                       'ATROPELLADO_FATAL_200',
                                       'FERROVIARIO_100',
                                       'FERROVIARIO_200',
                                       'MOTOCICLISTA_100',
                                       'MOTOCICLISTA_200',
                                       'OTROS_100',
                                       'OTROS_200',
                                       'PER_ATRAPADO_DESBARRANCADO_100',
                                       'PER_ATRAPADO_DESBARRANCADO_200',
                                       'VEH_ATRAPADO_100',
                                       'VEH_ATRAPADO_200',
                                       'VEH_DESBARRANCADO_100',
                                       'VEH_DESBARRANCADO_200',
                                       'VOLCADURA_100',
                                       'VOLCADURA_200',
                                       'SINIESTRO']]

train_dir_feat200 = train_dir_feat200[['ACC_C_LESION_100',
                                       'ACC_C_LESION_200',
                                       'ACC_FATAL_100',
                                       'ACC_FATAL_200',
                                       'ACC_PRENSADO_100',
                                       'ACC_PRENSADO_200',
                                       'ACC_S_LESION_100',
                                       'ACC_S_LESION_200',
                                       'ATROPELLADO_100',
                                       'ATROPELLADO_200',
                                       'ATROPELLADO_FATAL_100',
                                       'ATROPELLADO_FATAL_200',
                                       'FERROVIARIO_100',
                                       'FERROVIARIO_200',
                                       'MOTOCICLISTA_100',
                                       'MOTOCICLISTA_200',
                                       'OTROS_100',
                                       'OTROS_200',
                                       'PER_ATRAPADO_DESBARRANCADO_100',
                                       'PER_ATRAPADO_DESBARRANCADO_200',
                                       'VEH_ATRAPADO_100',
                                       'VEH_ATRAPADO_200',
                                       'VEH_DESBARRANCADO_100',
                                       'VEH_DESBARRANCADO_200',
                                       'VOLCADURA_100',
                                       'VOLCADURA_200',
                                       'SINIESTRO']]

train_dir_feat300 = train_dir_feat300[['ACC_C_LESION_100',
                                       'ACC_C_LESION_200',
                                       'ACC_FATAL_100',
                                       'ACC_FATAL_200',
                                       'ACC_PRENSADO_100',
                                       'ACC_PRENSADO_200',
                                       'ACC_S_LESION_100',
                                       'ACC_S_LESION_200',
                                       'ATROPELLADO_100',
                                       'ATROPELLADO_200',
                                       'ATROPELLADO_FATAL_100',
                                       'ATROPELLADO_FATAL_200',
                                       'FERROVIARIO_100',
                                       'FERROVIARIO_200',
                                       'MOTOCICLISTA_100',
                                       'MOTOCICLISTA_200',
                                       'OTROS_100',
                                       'OTROS_200',
                                       'PER_ATRAPADO_DESBARRANCADO_100',
                                       'PER_ATRAPADO_DESBARRANCADO_200',
                                       'VEH_ATRAPADO_100',
                                       'VEH_ATRAPADO_200',
                                       'VEH_DESBARRANCADO_100',
                                       'VEH_DESBARRANCADO_200',
                                       'VOLCADURA_100',
                                       'VOLCADURA_200',
                                       'SINIESTRO']]

train_dir_feat500 = train_dir_feat500[['ACC_C_LESION_100',
                                       'ACC_C_LESION_200',
                                       'ACC_FATAL_100',
                                       'ACC_FATAL_200',
                                       'ACC_PRENSADO_100',
                                       'ACC_PRENSADO_200',
                                       'ACC_S_LESION_100',
                                       'ACC_S_LESION_200',
                                       'ATROPELLADO_100',
                                       'ATROPELLADO_200',
                                       'ATROPELLADO_FATAL_100',
                                       'ATROPELLADO_FATAL_200',
                                       'FERROVIARIO_100',
                                       'FERROVIARIO_200',
                                       'MOTOCICLISTA_100',
                                       'MOTOCICLISTA_200',
                                       'OTROS_100',
                                       'OTROS_200',
                                       'PER_ATRAPADO_DESBARRANCADO_100',
                                       'PER_ATRAPADO_DESBARRANCADO_200',
                                       'VEH_ATRAPADO_100',
                                       'VEH_ATRAPADO_200',
                                       'VEH_DESBARRANCADO_100',
                                       'VEH_DESBARRANCADO_200',
                                       'VOLCADURA_100',
                                       'VOLCADURA_200',
                                       'SINIESTRO']]
test_dir_feat50 = test_dir_feat50[[    'ACC_C_LESION_100',
                                       'ACC_C_LESION_200',
                                       'ACC_FATAL_100',
                                       'ACC_FATAL_200',
                                       'ACC_PRENSADO_100',
                                       'ACC_PRENSADO_200',
                                       'ACC_S_LESION_100',
                                       'ACC_S_LESION_200',
                                       'ATROPELLADO_100',
                                       'ATROPELLADO_200',
                                       'ATROPELLADO_FATAL_100',
                                       'ATROPELLADO_FATAL_200',
                                       'FERROVIARIO_100',
                                       'FERROVIARIO_200',
                                       'MOTOCICLISTA_100',
                                       'MOTOCICLISTA_200',
                                       'OTROS_100',
                                       'OTROS_200',
                                       'PER_ATRAPADO_DESBARRANCADO_100',
                                       'PER_ATRAPADO_DESBARRANCADO_200',
                                       'VEH_ATRAPADO_100',
                                       'VEH_ATRAPADO_200',
                                       'VEH_DESBARRANCADO_100',
                                       'VEH_DESBARRANCADO_200',
                                       'VOLCADURA_100',
                                       'VOLCADURA_200',
                                       'SINIESTRO']]


test_dir_feat100 = test_dir_feat100[[  'ACC_C_LESION_100',
                                       'ACC_C_LESION_200',
                                       'ACC_FATAL_100',
                                       'ACC_FATAL_200',
                                       'ACC_PRENSADO_100',
                                       'ACC_PRENSADO_200',
                                       'ACC_S_LESION_100',
                                       'ACC_S_LESION_200',
                                       'ATROPELLADO_100',
                                       'ATROPELLADO_200',
                                       'ATROPELLADO_FATAL_100',
                                       'ATROPELLADO_FATAL_200',
                                       'FERROVIARIO_100',
                                       'FERROVIARIO_200',
                                       'MOTOCICLISTA_100',
                                       'MOTOCICLISTA_200',
                                       'OTROS_100',
                                       'OTROS_200',
                                       'PER_ATRAPADO_DESBARRANCADO_100',
                                       'PER_ATRAPADO_DESBARRANCADO_200',
                                       'VEH_ATRAPADO_100',
                                       'VEH_ATRAPADO_200',
                                       'VEH_DESBARRANCADO_100',
                                       'VEH_DESBARRANCADO_200',
                                       'VOLCADURA_100',
                                       'VOLCADURA_200',
                                       'SINIESTRO']]

test_dir_feat200 = test_dir_feat200[[  'ACC_C_LESION_100',
                                       'ACC_C_LESION_200',
                                       'ACC_FATAL_100',
                                       'ACC_FATAL_200',
                                       'ACC_PRENSADO_100',
                                       'ACC_PRENSADO_200',
                                       'ACC_S_LESION_100',
                                       'ACC_S_LESION_200',
                                       'ATROPELLADO_100',
                                       'ATROPELLADO_200',
                                       'ATROPELLADO_FATAL_100',
                                       'ATROPELLADO_FATAL_200',
                                       'FERROVIARIO_100',
                                       'FERROVIARIO_200',
                                       'MOTOCICLISTA_100',
                                       'MOTOCICLISTA_200',
                                       'OTROS_100',
                                       'OTROS_200',
                                       'PER_ATRAPADO_DESBARRANCADO_100',
                                       'PER_ATRAPADO_DESBARRANCADO_200',
                                       'VEH_ATRAPADO_100',
                                       'VEH_ATRAPADO_200',
                                       'VEH_DESBARRANCADO_100',
                                       'VEH_DESBARRANCADO_200',
                                       'VOLCADURA_100',
                                       'VOLCADURA_200',
                                       'SINIESTRO']]

test_dir_feat300 = test_dir_feat300[[  'ACC_C_LESION_100',
                                       'ACC_C_LESION_200',
                                       'ACC_FATAL_100',
                                       'ACC_FATAL_200',
                                       'ACC_PRENSADO_100',
                                       'ACC_PRENSADO_200',
                                       'ACC_S_LESION_100',
                                       'ACC_S_LESION_200',
                                       'ATROPELLADO_100',
                                       'ATROPELLADO_200',
                                       'ATROPELLADO_FATAL_100',
                                       'ATROPELLADO_FATAL_200',
                                       'FERROVIARIO_100',
                                       'FERROVIARIO_200',
                                       'MOTOCICLISTA_100',
                                       'MOTOCICLISTA_200',
                                       'OTROS_100',
                                       'OTROS_200',
                                       'PER_ATRAPADO_DESBARRANCADO_100',
                                       'PER_ATRAPADO_DESBARRANCADO_200',
                                       'VEH_ATRAPADO_100',
                                       'VEH_ATRAPADO_200',
                                       'VEH_DESBARRANCADO_100',
                                       'VEH_DESBARRANCADO_200',
                                       'VOLCADURA_100',
                                       'VOLCADURA_200',
                                       'SINIESTRO']]

test_dir_feat500 = test_dir_feat500[[  'ACC_C_LESION_100',
                                       'ACC_C_LESION_200',
                                       'ACC_FATAL_100',
                                       'ACC_FATAL_200',
                                       'ACC_PRENSADO_100',
                                       'ACC_PRENSADO_200',
                                       'ACC_S_LESION_100',
                                       'ACC_S_LESION_200',
                                       'ATROPELLADO_100',
                                       'ATROPELLADO_200',
                                       'ATROPELLADO_FATAL_100',
                                       'ATROPELLADO_FATAL_200',
                                       'FERROVIARIO_100',
                                       'FERROVIARIO_200',
                                       'MOTOCICLISTA_100',
                                       'MOTOCICLISTA_200',
                                       'OTROS_100',
                                       'OTROS_200',
                                       'PER_ATRAPADO_DESBARRANCADO_100',
                                       'PER_ATRAPADO_DESBARRANCADO_200',
                                       'VEH_ATRAPADO_100',
                                       'VEH_ATRAPADO_200',
                                       'VEH_DESBARRANCADO_100',
                                       'VEH_DESBARRANCADO_200',
                                       'VOLCADURA_100',
                                       'VOLCADURA_200',
                                       'SINIESTRO']]
train50 = pd.concat([ind_feat50,train_dir_feat50], axis=1)
train100 = pd.concat([ind_feat100,train_dir_feat100], axis=1)
train200 = pd.concat([ind_feat200,train_dir_feat200], axis=1)
train300 = pd.concat([ind_feat300,train_dir_feat300], axis=1)
train500 = pd.concat([ind_feat500,train_dir_feat500], axis=1)

test50 = pd.concat([ind_feat50,test_dir_feat50], axis=1)
test100 = pd.concat([ind_feat100,test_dir_feat100], axis=1)
test200 = pd.concat([ind_feat200,test_dir_feat200], axis=1)
test300 = pd.concat([ind_feat300,test_dir_feat300], axis=1)
test500 = pd.concat([ind_feat500,test_dir_feat500], axis=1)
train50.drop('Unnamed: 0', axis = 1, inplace = True)
train100.drop('Unnamed: 0', axis = 1, inplace = True)
train200.drop('Unnamed: 0', axis = 1, inplace = True)
train300.drop('Unnamed: 0', axis = 1, inplace = True)
train500.drop('Unnamed: 0', axis = 1, inplace = True)
test50.drop('Unnamed: 0', axis = 1, inplace = True)
test100.drop('Unnamed: 0', axis = 1, inplace = True)
test200.drop('Unnamed: 0', axis = 1, inplace = True)
test300.drop('Unnamed: 0', axis = 1, inplace = True)
test500.drop('Unnamed: 0', axis = 1, inplace = True)
train500.info()
test500.info()
train50.to_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/final_datasets/train50_fin.csv',encoding = 'utf-8')
train100.to_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/final_datasets/train100_fin.csv',encoding = 'utf-8')
train200.to_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/final_datasets/train200_fin.csv',encoding = 'utf-8')
train300.to_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/final_datasets/train300_fin.csv',encoding = 'utf-8')
train500.to_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/final_datasets/train500_fin.csv',encoding = 'utf-8')

test50.to_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/final_datasets/test50_fin.csv',encoding = 'utf-8')
test100.to_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/final_datasets/test100_fin.csv',encoding = 'utf-8')
test200.to_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/final_datasets/test200_fin.csv',encoding = 'utf-8')
test300.to_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/final_datasets/test300_fin.csv',encoding = 'utf-8')
test500.to_csv('/Users/daniel.rodriguez/Documents/ACC/ACC_PROOF/ACC1/final_datasets/test500_fin.csv',encoding = 'utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08372611192047305, 'fraction_numerical': 0.0543319907447082, 'mean_word_length': 1.0433374190159341, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 260, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '594335', 'license': 'MIT', 'n_tokens_mistral': 7840, 'n_tokens_neox': 6698, 'n_words': 648, 'path': 'CODE/py3/Create Final Dataset.ipynb', 'repo_name': 'Daro05/tesis_accidentes', 'size': '23338'}