<jupyter_start><jupyter_text>Reference1. https://www.kaggle.com/rohanrao/tutorial-on-reading-large-datasets1. https://www.kaggle.com/asobod11138/gsdc-neuralnet-keras (multi-threading)  Import Libraries<jupyter_code>import numpy as np
import pandas as pd
from glob import glob
import os
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
from pathlib import Path
import plotly.express as px
from multiprocessing import Pool
import multiprocessing as multi
import gc

from utils.gsdc_parser import ground_truth_file_open, derived_file_open, gnsslog_file_open, gnss_log_to_dataframes

split_charater = "\\"<jupyter_output><empty_output><jupyter_text>Help Function<jupyter_code>def UTC2GpsEpoch(df):
    '''UTC to GpsEpoch
    
    utcTimeMillis         : UTC epoch (1970/1/1)
    millisSinceGpsEpoch   : GPS epoch(1980/1/6 midnight 12:00 UTC)
    
    Ref: https://www.kaggle.com/c/google-smartphone-decimeter-challenge/discussion/239187
    '''
    dt_offset = pd.to_datetime('1980-01-06 00:00:00') 
    dt_offset_in_ms = int(dt_offset.value / 1e6)
    df['millisSinceGpsEpoch'] = df['utcTimeMillis'] - dt_offset_in_ms + 18000
    df['millisSinceGpsEpoch'] = df['millisSinceGpsEpoch'].astype('int64')
    return df<jupyter_output><empty_output><jupyter_text>Set Path and Load Dataset<jupyter_code>PATH = Path("../input/google-smartphone-decimeter-challenge")
train_df = pd.read_csv(PATH / "baseline_locations_train.csv")
test_df = pd.read_csv(PATH / "baseline_locations_test.csv")
print(train_df.shape)
train_df.head()
print(test_df.shape)
test_df.head()<jupyter_output>(91486, 7)<jupyter_text>Load All Data Function<jupyter_code>gnss_section_names = {'Raw','UncalAccel', 'UncalGyro', 'UncalMag', 'Fix', 'Status', 'OrientationDeg'}
section_names = {'GroundTruth', 'Derived', 'Raw','UncalAccel', 'UncalGyro', 'UncalMag', 'Fix', 'Status', 'OrientationDeg'}
_columns = ['latDeg', 'lngDeg', 'heightAboveWgs84EllipsoidM']
thread_num = 4
def merge_data(df_, truth, derived, gnss):
    df = df_.copy()
    
    print('Truth Data Start')
    if truth is not None:
        df = pd.merge_asof(df.sort_values('millisSinceGpsEpoch'), 
              truth.sort_values('millisSinceGpsEpoch'), 
              on="millisSinceGpsEpoch", by=["collectionName", "phoneName"], 
              direction='nearest',tolerance=100000)
    print('Truth Data Fin')

    print('Derived Data Start')
    df = pd.merge_asof(df.sort_values('millisSinceGpsEpoch'), 
          derived.sort_values('millisSinceGpsEpoch'), 
          on="millisSinceGpsEpoch", by=["collectionName", "phoneName"], 
          direction='nearest',tolerance=100000)
    print('Derived Data Fin')


    print('Gnss Log Data Start')
    for key, value in tqdm(gnss.items()):
        if value.shape[0] == 0:
            continue
        df = pd.merge_asof(df.sort_values('millisSinceGpsEpoch'), 
              value.sort_values('millisSinceGpsEpoch'), 
              on="millisSinceGpsEpoch", by=["collectionName", "phoneName"], 
              direction='nearest',tolerance=100000, suffixes = ['', "_" + key])
    print('Gnss Log Data Fin')
        
    return df<jupyter_output><empty_output><jupyter_text>Train For Multi Processing Code<jupyter_code>path_list = list(glob(str(PATH / "train" / "*/*/*")))

get_ground_truth = lambda x:x.split(split_charater)[-1].find("ground_truth")>= 0
get_derived = lambda x:x.split(split_charater)[-1].find("derived.csv")>= 0
get_gnsslog = lambda x:x.split(split_charater)[-1].find("GnssLog.txt")>= 0

ground_truth_path_list = list(filter(get_ground_truth, path_list))
derived_path_list = list(filter(get_derived, path_list))
gnsslog_path_list = list(filter(get_gnsslog, path_list))
output = dict()
for section in section_names:
    output[section] = pd.DataFrame()
with Pool(thread_num) as pool:
    imap = pool.imap(gnsslog_file_open, gnsslog_path_list)
    gnss_logs = list(tqdm(imap, total=len(gnsslog_path_list), desc="load gnss log"))
    
gnss_dict = {}
for key in gnss_section_names:
    list_temp = []
    for gnss_log in gnss_logs:
        list_temp.append(gnss_log[key])
    df_temp = pd.concat(list_temp)
    gnss_dict[key] = df_temp
    if (key == "Status") or (key == "Fix"):  
        gnss_dict[key].rename(columns = {'UnixTimeMillis':'utcTimeMillis'}, inplace = True)
    
    gnss_dict[key] = UTC2GpsEpoch(gnss_dict[key])
    print(gnss_dict[key]['utcTimeMillis'].dtype)
    print(gnss_dict[key].shape)
    print(gnss_dict[key].head())

del gnss_logs
with Pool(thread_num) as pool:
    imap = pool.imap(ground_truth_file_open, ground_truth_path_list)
    ground_truth = list(tqdm(imap, total=len(ground_truth_path_list), desc="load ground truth"))
    
df_truth = pd.concat(ground_truth)
print(df_truth.shape)
df_truth.head()

del ground_truth
with Pool(thread_num) as pool:
    imap = pool.imap(derived_file_open, derived_path_list)
    derived = list(tqdm(imap, total=len(derived_path_list), desc="load derived data"))
    
df_derived = pd.concat(derived)
print(df_derived.shape)
df_derived.head()

del derived<jupyter_output><empty_output><jupyter_text>Tiny Data Issue Solve<jupyter_code>raw_timestamps = gnss_dict['Raw']['millisSinceGpsEpoch'].unique()
derived_timestamps = df_derived['millisSinceGpsEpoch'].unique()

# The timestamps in derived are one epoch ahead. We need to map each epoch
# in derived to the prior one (in Raw).
indexes = np.searchsorted(raw_timestamps, derived_timestamps)
from_t_to_fix_derived = dict(zip(derived_timestamps, raw_timestamps[indexes-1]))
df_derived['millisSinceGpsEpoch'] = np.array(list(map(lambda v: from_t_to_fix_derived[v], df_derived['millisSinceGpsEpoch'])))
delta_millis = df_derived['millisSinceGpsEpoch'] - df_derived['receivedSvTimeInGpsNanos']/1e6
good_signals = (delta_millis > 0) & (delta_millis < 300)
df_derived = df_derived[good_signals].copy()<jupyter_output><empty_output><jupyter_text>Save To Pickle File<jupyter_code>output = merge_data(train_df, df_truth, df_derived, gnss_dict)
output.to_pickle(str(PATH / "gsdc_train.pkl.gzip"))
print(output.shape)
output.head()
output.info(verbose = True, memory_usage= True, null_counts=True)
del df_truth, df_derived, gnss_dict, output
gc.collect()<jupyter_output><empty_output><jupyter_text>Test For Multi Processing Code<jupyter_code>path_list = list(glob(str(PATH / "test" / "*/*/*")))

get_ground_truth = lambda x:x.split(split_charater)[-1].find("ground_truth")>= 0
get_derived = lambda x:x.split(split_charater)[-1].find("derived.csv")>= 0
get_gnsslog = lambda x:x.split(split_charater)[-1].find("GnssLog.txt")>= 0

ground_truth_path_list = list(filter(get_ground_truth, path_list))
derived_path_list = list(filter(get_derived, path_list))
gnsslog_path_list = list(filter(get_gnsslog, path_list))
output = dict()
for section in section_names:
    output[section] = pd.DataFrame()
with Pool(thread_num) as pool:
    imap = pool.imap(gnsslog_file_open, gnsslog_path_list)
    gnss_logs = list(tqdm(imap, total=len(gnsslog_path_list), desc="load gnss log"))
    
gnss_dict = {}
for key in gnss_section_names:
    list_temp = []
    for gnss_log in gnss_logs:
        list_temp.append(gnss_log[key])
    df_temp = pd.concat(list_temp)
    gnss_dict[key] = df_temp
    if (key == "Status") or (key == "Fix"):  
        gnss_dict[key].rename(columns = {'UnixTimeMillis':'utcTimeMillis'}, inplace = True)
    
    gnss_dict[key] = UTC2GpsEpoch(gnss_dict[key])
    print(gnss_dict[key]['utcTimeMillis'].dtype)
    print(gnss_dict[key].shape)
    print(gnss_dict[key].head())

del gnss_logs
with Pool(thread_num) as pool:
    imap = pool.imap(derived_file_open, derived_path_list)
    derived = list(tqdm(imap, total=len(derived_path_list), desc="load derived data"))
    
df_derived = pd.concat(derived)
print(df_derived.shape)
df_derived.head()

del derived<jupyter_output><empty_output><jupyter_text>Tiny Data Issue Solve<jupyter_code>raw_timestamps = gnss_dict['Raw']['millisSinceGpsEpoch'].unique()
derived_timestamps = df_derived['millisSinceGpsEpoch'].unique()

# The timestamps in derived are one epoch ahead. We need to map each epoch
# in derived to the prior one (in Raw).
indexes = np.searchsorted(raw_timestamps, derived_timestamps)
from_t_to_fix_derived = dict(zip(derived_timestamps, raw_timestamps[indexes-1]))
df_derived['millisSinceGpsEpoch'] = np.array(list(map(lambda v: from_t_to_fix_derived[v], df_derived['millisSinceGpsEpoch'])))
delta_millis = df_derived['millisSinceGpsEpoch'] - df_derived['receivedSvTimeInGpsNanos']/1e6
good_signals = (delta_millis > 0) & (delta_millis < 300)
df_derived = df_derived[good_signals].copy()<jupyter_output><empty_output><jupyter_text>Save To Pickle File<jupyter_code>output = merge_data(test_df, None, df_derived, gnss_dict)
output.to_pickle(str(PATH / "gsdc_test.pkl.gzip"))
print(output.shape)
output.head()
output.info(verbose = True, memory_usage= True, null_counts=True)
del df_derived, gnss_dict, output
%clear<jupyter_output><jupyter_text>Load Pickle File<jupyter_code>import numpy as np
import pandas as pd
from glob import glob
import os
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
from pathlib import Path
import plotly.express as px
PATH = Path("../input/google-smartphone-decimeter-challenge")
df_train = pd.read_pickle(str(PATH / "gsdc_train.pkl.gzip"))
print(df_train.shape)
df_train.head()
df_train.info(verbose = True, memory_usage= True, null_counts=True)
df_test = pd.read_pickle(str(PATH / "gsdc_test.pkl.gzip"))
print(df_test.shape)
df_test.head()
df_test.info(verbose = True, memory_usage= True, null_counts=True)
submission = pd.read_csv("../input/google-smartphone-decimeter-challenge/sample_submission.csv")
submission.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13003416502743556, 'fraction_numerical': 0.011388342478517445, 'mean_word_length': 5.769446391030133, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 51, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '249757', 'license': 'MIT', 'n_tokens_mistral': 3738, 'n_tokens_neox': 3538, 'n_words': 605, 'path': 'Research/GoogleSmartPhone/code/GSDC1_GatherData.ipynb', 'repo_name': 'leeh8911/BeSuperRepo', 'size': '9659'}