<jupyter_start><jupyter_text>Preprocessing PartAnnotation<jupyter_code>pc_part_anno_dir = "/home/ubuntu/datasets/full_shapenet_seg/03001627"
sn_list = [sn_id for sn_id in sn_model_to_int.keys()]
def read_point_cloud_part_anno(sn_id):
    pcd_path = osp.join(pc_part_anno_dir, f"{sn_id}.txt")
    pc = np.loadtxt(pcd_path)[:2048,[0,1,2,-1]]
    assert pc.shape[0] == 2048
    int_id = sn_model_to_int[sn_id]
    return pc, int_id
no_in_part_dataset_list = [] # 2038: doesn't exist in part datasets
exist_list = [] #2473
for sn in sn_list: #sn_list: 4511 
    pcd_path = osp.join(pc_part_anno_dir, f"{sn}.txt")
    if not os.path.exists(pcd_path):
        no_in_part_dataset_list.append(sn)
    else:
        exist_list.append(sn)
with open(osp.join(top_data_dir, "no_in_part_dataset_list.txt"), "w") as f:
    for x in no_in_part_dataset_list:
        f.write(f"{x} ")
chair_a = np.array(game_data.chair_a)
chair_b = np.array(game_data.chair_b)
chair_c = np.array(game_data.chair_c)
chairs = np.array(game_data[["chair_a", "chair_b", "chair_c"]])
no_in_part_dataset_set = set(no_in_part_dataset_int_list)
for i in range(10):
    chat_set = set(chairs[i])
    if chat_set.isdisjoint(no_in_part_dataset_set):
        pass # mask 1
    else:
        pass #mask 0 <- have at least one chair that doesn't exist in part dataset, so should be excluded.
    
from shapeglot.in_out.rnn_data_preprocessing import prepare_rnn_dataset
geo_ids, labels, padded_text, seq_len, mask = prepare_rnn_dataset(game_data, max_seq_len=33, only_correct=True, drop_too_long=True,
                    only_easy=True, part_anno=True)
from shapeglot.in_out.rnn_data_preprocessing import make_dataset_for_rnn_based_model

net_data, split_ids, _, net_data_mask = \
make_dataset_for_rnn_based_model(game_data, 
                                 splits=[0.8, 0.1, 0.1], 
                                 max_seq_len=33,
                                 drop_too_long=True,
                                 only_correct=True,
                                 unique_test_geo=True,
                                 replace_not_in_train=True,
                                 geo_condition=None, 
                                 bias_train=False, 
                                 seed=2004,
                                only_easy=True,
                                part_anno=True) 
len(net_data["train"])
net_data2, split_ids2, _, net_data_mask2 = \
make_dataset_for_rnn_based_model(game_data, 
                                 splits=[0.8, 0.1, 0.1], 
                                 max_seq_len=33,
                                 drop_too_long=True,
                                 only_correct=True,
                                 unique_test_geo=True,
                                 replace_not_in_train=True,
                                 geo_condition=None, 
                                 bias_train=False, 
                                 seed=2004,
                                only_easy=True,
                                part_anno=False) 
len(net_data2["train"])
pool = Pool(processes=8)
results = pool.map(read_point_cloud_part_anno, exist_list)
results=np.array(results)
pcs_with_part_anno = np.array(list(results[:,0])).astype(np.float32)
ids = np.array(results[:,1]).astype('int64')
print(pcs.shape, pcs.dtype, ids.shape, ids.dtype)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09619952494061758, 'fraction_numerical': 0.021971496437054632, 'mean_word_length': 2.057168784029038, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '66818', 'license': 'MIT', 'n_tokens_mistral': 1179, 'n_tokens_neox': 1119, 'n_words': 170, 'path': 'transglot/notebooks/preprocess_point_cloud.ipynb', 'repo_name': '63days/Transglot', 'size': '3368'}