<jupyter_start><jupyter_text>Import<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
!pip install geopandas
import pandas as pd
import numpy as np
import itertools
from operator import itemgetter

import geopandas as gpd
import numpy as np
import pandas as pd

from scipy.spatial import cKDTree
from shapely.geometry import Point, LineString
road_data_path = '/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/route_network/clustered'
trips_data_path = '/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/trips_and_stops_NE_2019'
export_data_path = '/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/trips_with_wayID'
def ckdnearest(gdfA, gdfB, gdfB_cols=['way_id']):
    A = np.concatenate(
        [np.array(geom.coords) for geom in gdfA.geometry.to_list()])
    B = [np.array(geom.coords) for geom in gdfB.geometry.to_list()]
    B_ix = tuple(itertools.chain.from_iterable(
        [itertools.repeat(i, x) for i, x in enumerate(list(map(len, B)))]))
    B = np.concatenate(B)
    ckd_tree = cKDTree(B)
    dist, idx = ckd_tree.query(A, k=1)
    idx = itemgetter(*idx)(B_ix)
    gdf = pd.concat(
        [gdfA, gdfB.loc[idx, gdfB_cols].reset_index(drop=True),
         pd.Series(dist, name='dist')], axis=1)
    return gdf<jupyter_output><empty_output><jupyter_text>Get way sequence<jupyter_code>days = range(11,32)
month = 8
for day in days:
  export_filename = export_data_path + f'/trips_with_wayId_{month}_{day}.csv'
  trips_df = pd.read_csv(trips_data_path+f'/trips_2019_{month}_{day}.csv', parse_dates=['time_stamp'])
  way_segment_drop_dup_df = pd.read_csv(road_data_path + 'way_and_road_segment_lazy_drop.csv')
  # from pd to gpd
  trips_gdf = gpd.GeoDataFrame(
    trips_df, geometry=gpd.points_from_xy(trips_df.lat, trips_df.lon))

# create linestring of way_id
  way_segment_drop_dup_gdf = gpd.GeoDataFrame(
  way_segment_drop_dup_df, geometry=gpd.points_from_xy(
      way_segment_drop_dup_df.mid_lat,
      way_segment_drop_dup_df.mid_lon
    )
  )

  way_id_list = []
  for way_id in pd.unique(way_segment_drop_dup_gdf.way_id):
    line = way_segment_drop_dup_gdf[way_segment_drop_dup_gdf.way_id == way_id]['geometry'].to_list()
    if len(line) < 2:
      line = line*2
    way_id_list.append([way_id, LineString(line)])
  way_id_list

  way_segment_drop_dup_gdf = gpd.GeoDataFrame(way_id_list, columns=['way_id', 'geometry'])

  trips_gdf_nearest_way = ckdnearest(trips_gdf, way_segment_drop_dup_gdf)

  export_df = trips_gdf_nearest_way.drop(columns=['geometry'])
  export_df.to_csv(export_filename, index=False)
  print(export_filename)<jupyter_output>/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/trips_with_wayID/trips_with_wayId_8_11.csv
/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/trips_with_wayID/trips_with_wayId_8_12.csv
/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/trips_with_wayID/trips_with_wayId_8_13.csv
/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/trips_with_wayID/trips_with_wayId_8_14.csv
/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/trips_with_wayID/trips_with_wayId_8_15.csv
/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/trips_with_wayID/trips_with_wayId_8_16.csv
/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/trips_with_wayID/trips_with_wayId_8_17.csv
/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data/trips_with_wayID/trips_with_wayId_8_18.csv
/content/drive/My Drive/2021 Route Prediction/Project-1/Source-Code/data[...]
{'fraction_non_alphanumeric': 0.11335149863760217, 'fraction_numerical': 0.027520435967302453, 'mean_word_length': 5.440350877192983, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '433343', 'license': 'MIT', 'n_tokens_mistral': 1498, 'n_tokens_neox': 1445, 'n_words': 205, 'path': 'phase3/map_trips_to_way_sequence/get_way_sequence_loop.ipynb', 'repo_name': 'myNameIsPK/route_prediction', 'size': '3670'}