<jupyter_start><jupyter_text>Inference This notebook runs inference with the trained UNet model over. We store resutls for each location as a pickle file, containing the matrix of probabilities for each image.<jupyter_code>model = UNet(4, 2, full_size=True)
model.load_state_dict(torch.load('../local/unet_checkpoint.pth'))
out_dir = '../local/inference_results'
if not os.path.exists(out_dir)
    os.makedirs(out_dir)
truth_df = pd.read_csv('../local/truth.csv')
truth_df.head()
dim = 200 


for i, row in tqdm(truth_df.iterrows()): 
    
    if row.label == 2 or \
        os.path.exists(os.path.join(out_dir, f'{row.location_name}.p')): 
        continue 
    
    # sort images so they're in proper order
    ims = sorted(glob.glob(os.path.join(row.dir, '*.tif')))
    
    img_array = np.empty((len(ims), 4, dim, dim))
    for i, im in enumerate(ims): 
    
        with rasterio.open(im) as src:
            b, g, r, n = src.read()

        stack = (r,g,b,n)
        rgb = np.stack(stack, axis=0)
        np_image = ((rgb/rgb.max())*255).astype(np.uint8)
        img_array[i, :, :, :] = np_image
        
    # Apply model to image array         
    output = model(torch.Tensor(img_array)) 
    output = -output.detach().numpy().reshape(
        output.shape[0], 2,dim,dim
    )[:,1,:,:]
    probs = 1 / (1 + np.exp(output))    
    
    # Save output
    with open(os.path.join(out_dir, f'{row.location_name}.p'), 'wb') as f: 
        pickle.dump(probs, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15327978580990628, 'fraction_numerical': 0.011378848728246318, 'mean_word_length': 2.7848101265822787, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '263135', 'license': 'MIT', 'n_tokens_mistral': 575, 'n_tokens_neox': 545, 'n_words': 120, 'path': 'notebooks/inference.ipynb', 'repo_name': 'reglab/building-expansion', 'size': '1494'}