<jupyter_start><jupyter_text>Imports<jupyter_code>import pdb
import numpy as np
import os, shutil
from tqdm import tqdm_notebook
import pandas as pd
import open3d as o3d
import gc
import zipfile
from multiprocessing import Pool, cpu_count
# from IPython.core.display import display, HTML
# display(HTML("<style>.container { width:80% !important; }</style>"))<jupyter_output><empty_output><jupyter_text>Set Paths and constants<jupyter_code># pcd_folder='/home/sabyasachi/Projects/ati/data/data/datasets/Carla/lidarParam1/pair_corrupt/dynamic/'
# npy_folder='/home/sabyasachi/Projects/ati/data/data/datasets/Carla/lidarParam1/pair_corrupt/dynamic_NPY/'

BASE_PATH = '/home/sabyasachi/Projects/ati/data/data/datasets/Carla/64beam-Data'
PAIR_FOLDER = "pair"
# Do for both
PCD_FOLDER = "static"
# PCD_FOLDER = "dynamic"
EXTRACTED_ARRAY_FNAME = "arr_0.npy"

# BATCH_SIZE = 44000
BATCH_SIZE = 2048
RANGE_IMAGE_HEIGHT = 64
RANGE_IMAGE_WIDTH = 512
PCD_PATH = os.path.join(BASE_PATH, PAIR_FOLDER, PCD_FOLDER)
if not os.path.exists(PCD_PATH):
    print("Did not find : {}".format(PCD_PATH))

INITIAL_NPY_FOLDER = PCD_FOLDER + "_begin_npy"
INITIAL_NPY_PATH = os.path.join(BASE_PATH, PAIR_FOLDER, INITIAL_NPY_FOLDER)
if not os.path.exists(INITIAL_NPY_PATH):
    os.makedirs(INITIAL_NPY_PATH)
else:
    shutil.rmtree(INITIAL_NPY_PATH)
    os.makedirs(INITIAL_NPY_PATH)

NPZ_FOLDER = PCD_FOLDER + "_npz"
NPZ_PATH = os.path.join(BASE_PATH, PAIR_FOLDER, NPZ_FOLDER)
if not os.path.exists(NPZ_PATH):
    os.makedirs(NPZ_PATH)
else:
    shutil.rmtree(NPZ_PATH)
    os.makedirs(NPZ_PATH)

OUT_NPY_FOLDER = PCD_FOLDER + "_out_npy"
OUT_NPY_PATH = os.path.join(BASE_PATH, PAIR_FOLDER, OUT_NPY_FOLDER)
if not os.path.exists(OUT_NPY_PATH):
    os.makedirs(OUT_NPY_PATH)
else:
    shutil.rmtree(OUT_NPY_PATH)
    os.makedirs(OUT_NPY_PATH)<jupyter_output><empty_output><jupyter_text>Preprocessing functions from the paper source code<jupyter_code>def get_quadrant(point):
    if point[0] >= 0. and point[1] >= 0. :
        return 0
    elif point[0] <= 0. and point[1] >= 0. : 
        return 1
    elif point[0] <= 0. and point[1] <= 0. : 
        return 2
    elif point[0] >= 0. and point[1] <= 0. : 
        return 3
    else :
        raise Exception('invalid input %s', point) 


def passed_origin(x_t, x_t1):
    if get_quadrant(x_t1) == 3 and get_quadrant(x_t) == 0: 
        return True
    else : 
        return False


def fit_quadrant(points, quadrant, desired_amt):
    
    
    points = np.asarray(points)
    slots = []
    slot_size = np.pi / (2 * desired_amt)
    for i in range(int(desired_amt)) : slots.append([])
    if quadrant == 0: 
        points = points[::-1]
    elif quadrant == 1 : 
        points[:, 0] = - points[:, 0]
    elif quadrant == 2 :
        points = points[::-1] 
        points[:, 0] = - points[:, 0]
        points[:, 1] = - points[:, 1]
    elif quadrant == 3 : 
        points[:, 1] = - points[:, 1]

    # import pdb; pdb.set_trace()
    for point in points :
        angle = np.arctan(point[1] / (point[0]+0.000001))
        index = min(int(angle / slot_size), desired_amt - 1)
        slots[int(index)].append(point)

    for i in range(len(slots)):
        if len(slots[i]) == 0 : 
            slots[i] = np.array([0., 0., 0., 0.])
        else :
            full_slot = np.asarray(slots[i])
            slots[i] = full_slot.mean(axis=0)

    points = np.asarray(slots)
    if quadrant == 0: 
        points = points[::-1]
    elif quadrant == 1 : 
        points[:, 0] = - points[:, 0]
    elif quadrant == 2 : 
        points = points[::-1]
        points[:, 0] = - points[:, 0]
        points[:, 1] = - points[:, 1]
    elif quadrant == 3 : 
        points[:, 1] = - points[:, 1]

    return points

def parse_velo(velo):
    # points closer to the origin (0,0,0) are at the end of the point cloud.
    # invert the point cloud such that we begin near the origin. 
    
    # returns: a H x 4 x ? array, split into quadrants
    velo = velo[::-1]
    lines = []
    current_point = velo[0]
    current_quadrant = get_quadrant(current_point)
    current_line = [[], [], [], []]
    quadrant_switches = 0
    for point in velo :
        point_quadrant = get_quadrant(point)
        
        if passed_origin(current_point, point):
            lines.append(current_line)
            current_line = [[], [], [], []]

        current_line[point_quadrant].append(point)
        current_quadrant = point_quadrant
        current_point = point

    return lines


def setmatch(lines,lenLines):
    arr=[[np.array([0,0,0,0]),np.array([0,0,0,0])]]
    if len(lines) > lenLines:
        return lines[:lenLines]
    else:
        for i in range(abs(len(lines)-lenLines)):
            lines.append(arr)
    return lines

def process_velo(velo, points_per_layer, stop=False):
    
    lenLines=RANGE_IMAGE_HEIGHT
    lines = parse_velo(velo)
    inverse = quad_to_pc_inv(lines)
    lines = lines[2:-1]
#     print(lines[])
#     print((lines[0]))
#     raise SystemError
    if(len(lines)!=lenLines):
        lines=setmatch(lines,lenLines)
#     print(len(lines), flush=True)
    if len(lines) != RANGE_IMAGE_HEIGHT : raise Exception('invalid nb un of lines')
    out_tensor = np.zeros((RANGE_IMAGE_HEIGHT, points_per_layer, 4))
    if stop:
        import pdb; pdb.set_trace()
        x = 1
    for j in range(len(lines)):
        line = lines[j]
        out_line = np.zeros((points_per_layer, 4))
        for i in range(len(line)):
            if(len(line[i])==0):
                line[i]=[np.array([0.0,0.0,0.0,0.0])]
            gridded = fit_quadrant(line[i], i, points_per_layer / 4)
            out_tensor[j][i*int(points_per_layer/4):(i+1)*int(points_per_layer/4), :] = gridded[::-1]

    return out_tensor, inverse


def quad_to_pc_inv(lines, th=3.):
    # lines is a 63 x 4 array, where each slot has an array of 4d/3d points
    # goal : get an array of points that fills empty spaces
    points = []
    for i in range(len(lines)) :
        line = lines[i] 
        distance = []
        for quad in line : 
            for point in quad : 
                x, y, z = point[:3]
                distance.append(x**2 + y**2)
        distance = np.array(distance)
        std = distance.std()
        sorted_indices = np.argsort(distance)
        median_index = sorted_indices[int(sorted_indices.shape[0]*0.95)]
        median = distance[median_index]

        for quad in line : 
            for point in quad : 
                x, y, z = point[:3]
                dist = x ** 2 + y ** 2 
                if dist < median and (median/dist-1.) > th:#*std : 
                    # blocked point --> scale to get real pt
                    scale = np.sqrt(median / dist)
                    scaled = scale * point
                    points.append(scaled)


    return np.array(points)

def getint(name):
    return int(name.split('.')[0])<jupyter_output><empty_output><jupyter_text>PCD to NPY files<jupyter_code>def parallel_pcd2begin_npy(pcd_fname):
    pcd_file_path = os.path.join(PCD_PATH, pcd_fname)
    pcd = o3d.read_point_cloud(pcd_file_path)
    pcd_arr = np.asarray(pcd.points)
    pcd_arr = np.append(pcd_arr, np.zeros((pcd_arr.shape[0],1)), axis=1)
    
    npy_fname = pcd_fname[:-4] + ".npy"
    npy_file_path = os.path.join(INITIAL_NPY_PATH, npy_fname)
    pcd_arr.dump(open(npy_file_path, 'wb'))

parallel_npy_args = sorted(os.listdir(PCD_PATH), key=getint)
process_npy_pool = Pool(cpu_count()-1)
__ = [each for each in tqdm_notebook(process_npy_pool.imap(parallel_pcd2begin_npy,
                                                       parallel_npy_args),
                                     total = len(parallel_npy_args))]
process_npy_pool.terminate()
gc.collect()<jupyter_output><empty_output><jupyter_text>NPY to NPZ files<jupyter_code>def parallel_npy2processed(npy_file):
    gc.collect()
    npy_file_path = os.path.join(INITIAL_NPY_PATH, npy_file)
    raw_lidar = np.load(npy_file_path, allow_pickle=True)
    processed_lidar, _ = process_velo(raw_lidar, RANGE_IMAGE_WIDTH)
    return processed_lidar
npy_folder_size = len(os.listdir(INITIAL_NPY_PATH))
leftout_size = npy_folder_size % BATCH_SIZE
n_batches = int(npy_folder_size / BATCH_SIZE)
file_list = sorted(os.listdir(INITIAL_NPY_PATH), key=getint)
full_npy_file_list = np.split(np.array(file_list)[:-leftout_size], n_batches)
# To consider last small batch
full_npy_file_list += [np.array(file_list[-leftout_size:])]

print(len(full_npy_file_list))
npz_file_idx = 0
for some_npy_file_list in full_npy_file_list:
    print(len(some_npy_file_list))
    parallel_processed_args = some_npy_file_list
    process_processed_pool = Pool(cpu_count()-1)
    one_run_npy_file = [each for each in tqdm_notebook(process_processed_pool.imap(parallel_npy2processed,
                                                           parallel_processed_args), total=len(parallel_processed_args))]
    process_processed_pool.terminate()
    gc.collect()
    
    npz_file_path = os.path.join(NPZ_PATH, str(npz_file_idx))
    np.savez(npz_file_path, one_run_npy_file)
    npz_file_idx += 1<jupyter_output>2048<jupyter_text>NPZ extracted serially to NPY<jupyter_code>for npz_fname in tqdm_notebook(sorted(os.listdir(NPZ_PATH), key=getint)):
    npz_path = os.path.join(NPZ_PATH, npz_fname)
    with zipfile.ZipFile(npz_path, 'r') as zip_ref:
        zip_ref.extractall(OUT_NPY_PATH)
    
    out_npy_fname = npz_fname[:-4] + ".npy"
    src_fname = os.path.join(OUT_NPY_PATH, EXTRACTED_ARRAY_FNAME)
    dst_fname = os.path.join(OUT_NPY_PATH, out_npy_fname)
    os.rename(src_fname, dst_fname)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1243096801083672, 'fraction_numerical': 0.01698447431489007, 'mean_word_length': 3.0108650229837024, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '161736', 'license': 'MIT', 'n_tokens_mistral': 3718, 'n_tokens_neox': 3581, 'n_words': 675, 'path': 'Data_Generation/4_ply2processednpy.ipynb', 'repo_name': 'prashkmr/DSLR-Cyc', 'size': '9597'}