<jupyter_start><jupyter_text>Id Based on Yes Chips<jupyter_code>def get_chip_count(chip_file_loc):
    
    chip_count = {}
    
    with open(chip_file_loc) as r:
        
        chips = json.load(r)
        
    for chip in chips:

        poly_id = int(chip.split("/")[5])


        if poly_id in chip_count.keys():

            chip_count[poly_id] += 1 

        else:

            chip_count[poly_id] = 1 

    return chip_count
    
chip_file_loc = "yes_chips_s3.json"

chip_count = get_chip_count(chip_file_loc)
sum(chip_count.values())
def train_test_split(regions_loc,polygons_loc,chip_count,skip_poly_ids=[]):
    
    with open(regions_loc) as r, open(polygons_loc) as p:
        
        pol_groups = {}
        
        regions = json.load(r)["features"]
        polygons = json.load(p)["features"]
        
        r_col = GeometryCollection([shape(feature["geometry"]).buffer(0) for feature in regions])
        p_col = GeometryCollection([shape(feature["geometry"]).buffer(0) for feature in polygons])
        
        poly_list = skip_poly_ids.copy()
        
        for i,poly_1 in enumerate(r_col,1):
            
            
            for j, poly_2 in enumerate(p_col,1):
                
                
                if j not in poly_list:
                    

                    if poly_1.intersects(poly_2):
                        
                        poly_list.append(j)
                        

                        if i in list(pol_groups.keys()):

                            pol_groups[i].append({j:chip_count[j]})


                        else:

                            pol_groups[i] = [{j:chip_count[j]}]
                        
    
        return pol_groups
    
    
    
    
    
regions_loc = "/Volumes/Lacie/zhenyadata/Project_Canopy_Data/PC_Data/Geometry/custom_congo_basin_regions/congo_basin_custom_regions.geojson"
polygons_loc = "/Volumes/Lacie/zhenyadata/Project_Canopy_Data/PC_Data/Geometry/labelled_boxes/v3/labels.geojson"

pol_groups = train_test_split(regions_loc,polygons_loc,chip_count,skip_poly_ids=[95,96,97,98])
sum_chips = 0 

for k1 in pol_groups.keys():
    
    for k2 in pol_groups[k1]: 
        
        for v1 in k2.values():
            
            sum_chips += v1
        
sum_chips
train_test = {"train":[],"test":[]}

counter = 0 

for r_id in pol_groups.keys():
    
    vals_1 = []

    for p_list in pol_groups[r_id]:
        
        vals_1.append(list(p_list.values())[0])
        
    sum_vals = sum(vals_1)
    
    r_thresh = sum_vals * .84

    pol_groups_sorted = sorted(pol_groups[r_id],key=lambda i:list(i.values())[0],reverse=True)
    
    temp_sum = 0 
    
    
    for p_list in pol_groups_sorted:
        
        counter += 1 
        
        temp_sum += list(p_list.values())[0]
        
        if temp_sum < r_thresh:
            
            train_test["train"].append(list(p_list.keys())[0]) 
            
        else:
            
            train_test["test"].append(list(p_list.keys())[0]) 
                
train_test
pols = []
for k,v in pol_groups.items():  
    for p in v:
        pols.append(p)
tot_train = 0
tot_test = 0
for pol_id in train_test["train"]:
    for pol_dict in pols:
        if pol_id in pol_dict.keys():
            tot_train += pol_dict[pol_id]

for pol_id in train_test["test"]:
    for pol_dict in pols:
        if pol_id in pol_dict.keys():
            tot_test += pol_dict[pol_id]
tot_train
tot_test
tot_test / (tot_train + tot_test)
import json
with open('train_test_polygons.json', 'w') as fp:
    json.dump(train_test, fp)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10599334073251943, 'fraction_numerical': 0.01193118756936737, 'mean_word_length': 1.858842188739096, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '141078', 'license': 'MIT', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1211, 'n_words': 208, 'path': 'data-prep/ML_split.ipynb', 'repo_name': 'Project-Canopy/SSRC_New_Model_Development', 'size': '3604'}