<jupyter_start><jupyter_text>Clean Bad TilesWe're having an issue where lots of image tiles have no data. Let's clean them up.<jupyter_code>tilefolder_re = "(.*)CO(.*)"
tile_re = ".*_.*\/\d*\/\d*\/\d*\.tif"
fs = s3fs.S3FileSystem(session = boto3.Session(profile_name = 'esip'))
tileFolderCandidates = fs.ls('planet-snowcover-snow', update=True)
tileFolderCandidates
tileFolders = [f for f in tileFolderCandidates if match(tilefolder_re, f)]
allTiles = []
for folder in tqdm.tqdm(tileFolders, unit='folder'):
    allfiles = fs.walk(folder)
    tiles = [t for t in allfiles if match(tile_re, t)]
    allTiles.append(tiles)
allTiles = list(chain(*allTiles))
snowTiles = allTiles
imageTiles = allTiles
snowTiles = allTiles
len(snowTiles), len(imageTiles)
snowTileIds = ["/".join(t.split("/")[-3:]) for t in snowTiles]
imageTileIds = ["/".join(t.split("/")[-3:]) for t in imageTiles]
set(snowTileIds).intersection(imageTileIds)<jupyter_output><empty_output><jupyter_text>---<jupyter_code>def partialImageryTile(tilepath):
    with rio.Env(profile_name='esip'):
        try: 
            data = rio.open('s3://' + tilepath).read()
        except rio.errors.RasterioIOError as e:
            print(e)
            return False 
        
    flat = data.flatten()
    return flat.shape[0] != np.count_nonzero(flat)

def partialMaskTile(tilepath):
    with rio.Env(profile_name='esip'):
        try: 
            data = rio.open('s3://' + tilepath).read()
        except rio.errors.RasterioIOError as e:
            print(e)
            return False 
        
    flat = data.flatten()
    return flat.shape[0] != np.sum(flat >= 0)
    

        
len(allTiles)
p = Pool(processes = 10)
result = p.map(partialMaskTile, allTiles)
allTilesArray = np.array(allTiles)
partialIndices = np.where(np.array(result) == True)[0]
partialTiles = allTilesArray[partialIndices]
len(partialTiles), len(allTiles)
partialTiles
def mvPartialTile(tile):
    base, ext = os.path.splitext(tile)
    newBase = base+'_partialtile'
    fs.mv(tile, newBase + ext)
errored = 0
for tile in tqdm.tqdm(partialTiles, unit='tile'):
    try: 
        mvPartialTile(tile)
    except Exception as e: 
        errored += 1
        continue
        
print(errored)

    
partialTiles[0]
with rio.Env(profile_name='esip'):
    rio.open("s3://planet-snowcover-imagery/20180422_180809_1033_3B_AnalyticMS_SR_clip_tiled/15/5526/12722_partialtile.tif").read()<jupyter_output><empty_output><jupyter_text>Examine Full Tiles<jupyter_code>fullTiles = list(set(allTiles) - set(partialTiles))
tileSamp = sample(list(fullTiles), 5)
grid = plt.GridSpec(3, 3)
fig = plt.figure(figsize=(20,20), dpi = 150)
with rio.Env(profile_name="esip"):
    for i, tile in enumerate(tileSamp):
        t = rio.open("s3://" + tile)
        ax = plt.subplot(grid[i])
#         ax.imshow(np.moveaxis(np.stack([t.read(3), t.read(2), t.read(1)]), 0, -1))
        da = t.read(1)
        ax.imshow(da)
        ax.set_title((da.min(), da.max()))
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>--- Diagnostics<jupyter_code># should have some partial tiles, we don't want 'em
tiledir = "planet-snowcover-imagery/20180422_180809_1033_3B_AnalyticMS_SR_clip_tiled"
maskdir = "planet-snowcover-snow/ASO_3M_SD_USCAJW_20180423/ASO_3M_SD_USCAJW_20180423-MASK_02-COG"
from robosat_pink.datasets import MultiSlippyMapTilesConcatenation
mt = MultiSlippyMapTilesConcatenation([tiledir], [maskdir], aws_profile='esip')

partialTile('planet-snowcover-imagery/20180422_180809_1033_3B_AnalyticMS_SR_clip_tiled/15/5540/12721.tif')
list(partialTiles.values)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14281744929146986, 'fraction_numerical': 0.0369547096415671, 'mean_word_length': 4.164992826398852, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '395559', 'license': 'MIT', 'n_tokens_mistral': 1473, 'n_tokens_neox': 1379, 'n_words': 228, 'path': 'pipeline/machinery/X_remove_bad_tiles.ipynb', 'repo_name': 'acannistra/planet-snowcover', 'size': '3599'}