<jupyter_start><jupyter_text>Load the sample data   * Sampled small-scale complete data (trave survey data from 10,000 individuals)   * Sampled large-scale incomplete data (smart card data from 10,000 + 10,000 individuals)<jupyter_code>import numpy as np
import pandas as pd

#Load the sampled complete and incomplte data 
# x_train_cond_R = pd.read_csv('01_Data/train_complete_qualitative.csv')
# y_train_cond_R = pd.read_csv('01_Data/train_complete_tripChain.csv')
# y_train_cond_SC_R = pd.read_csv('01_Data/train_incomplete_tripChain.csv')

x_train_cond_R = pd.read_csv('Data/x_train_cond_R_Coord_ID_DisR.csv')
y_train_cond_R = pd.read_csv('Data/y_train_cond_R_Coord_ID_DisR.csv')
y_train_cond_SC_R = pd.read_csv('Data/y_train_cond_SC_R_Coord_ID_DisR_LatLon.csv')
print(x_train_cond_R.shape)
print(y_train_cond_R.shape)
print(y_train_cond_SC_R.shape)<jupyter_output>(64299, 9)
(64299, 20)
(262849, 23)<jupyter_text>Data Preprocessing * Transforming the long-form data into ndarray sequential form<jupyter_code>x_train_cond_R = x_train_cond_R[y_train_cond_R['Max_seq'] != 6]
y_train_cond_R = y_train_cond_R[y_train_cond_R['Max_seq'] != 6]
y_train_cond_SC_R = y_train_cond_SC_R[y_train_cond_SC_R['Max_seq'] != 6]
Sam_ID = x_train_cond_R['ID'].unique()
Sam_ID_SC_train = y_train_cond_SC_R['NID'].unique()[50000:100000]

x_train_cond_R_sample = x_train_cond_R[x_train_cond_R['ID'].isin(Sam_ID)]
y_train_cond_R_sample = y_train_cond_R[x_train_cond_R['ID'].isin(Sam_ID)]
y_train_cond_SC_R_sample = y_train_cond_SC_R[y_train_cond_SC_R['NID'].isin(Sam_ID_SC_train)]

x_train_cond_R_sample.to_csv('Data/train_complete_qualitative.csv',index=False)
y_train_cond_R_sample.to_csv('Data/train_complete_tripChain.csv',index=False)
y_train_cond_SC_R_sample.to_csv('Data/train_incomplete_tripChain_LatLon.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11038607939097335, 'fraction_numerical': 0.030451332245785754, 'mean_word_length': 10.5, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '567839', 'license': 'MIT', 'n_tokens_mistral': 821, 'n_tokens_neox': 764, 'n_words': 89, 'path': '.ipynb_checkpoints/DataPreprocessing_Backup-checkpoint.ipynb', 'repo_name': 'Eui-Jin/Visual-Unfolded', 'size': '1839'}