<jupyter_start><jupyter_text>Doing the actual thing<jupyter_code>class_np_list = sorted(glob.glob('/srv/shared/planetpieces/training/*.npy'))
ar = np.load(class_np_list[0])
len(blockshaped(ar,64,64))
def save_chunks(input_list):
    """
    input list is [npy,fn,i]
    """
    npy = input_list[0][0]
    fn = input_list[0][1]
    i = input_list[1]
    outfn = os.path.splitext(fn)[0]+'_chunk_{}.npy'.format(i)
    np.save(outfn,npy)
def chunk_class(outfolder,fn):
    np_arr = np.load(fn)
    band_chunked = blockshaped(np_arr,64,64)
    outfn = os.path.join(outfolder,(os.path.basename(fn)))
    fn_list = [outfn]*len(band_chunked)
    save_list = list(zip(band_chunked,fn_list))
    save_list = list(zip(save_list,np.arange(len(band_chunked))))
    with Pool(cpu_count()*2) as p:
        p.map(save_chunks, [chunks for chunks in save_list])
    #return save_list
    #for i in range(len(band_chunked)):
     #   outfn = os.path.splitext(fn)[0]+'_chunk_{}.npy'.format(i)
      #  np.save(outfn,band_chunked[i])
chunk_class(class_np_list[0])
test = chunk_class(class_np_list[0])
os.path.join(outfolder,(os.path.basename(class_np_list[0])))
outfolder = '/srv/shared/planetpieces/chunks/class/'
for classified in class_np_list:
    chunk_class(outfolder,classified)
def parallel_chunk_bands(outfolder,fn):
    ds = rasterio.open(fn)
    band_list = [np.ma.masked_equal(ds.read(x),0) for x in range(1,5)]
    band_chunk_list = [blockshaped(band,64,64) for band in band_list]
    out_stack = np.stack(band_chunk_list,axis=0)
    outfn = os.path.splitext(os.path.join(outfolder,os.path.basename(fn)))[0]+'_img.npy'
    fn_list = [outfn]*np.shape(out_stack)[1]
    list_out = [out_stack[:,i,:,:] for i in range(np.shape(out_stack)[1])]
    save_list = list(zip(list_out,fn_list))
    save_list = list(zip(save_list,np.arange(np.shape(out_stack)[1])))
    with Pool(cpu_count()*4) as p:
        p.map(save_chunks, [chunks for chunks in save_list])

    
    return band_chunk_list<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16117764471057885, 'fraction_numerical': 0.016467065868263474, 'mean_word_length': 4.845481049562682, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '584130', 'license': 'MIT', 'n_tokens_mistral': 826, 'n_tokens_neox': 818, 'n_words': 111, 'path': 'contributors/shashank/chunking_shashank.ipynb', 'repo_name': 'geohackweek/ghw2019_planetpieces', 'size': '2004'}