<jupyter_start><jupyter_text>Define useful functions<jupyter_code>def find_world_diffs(df_proc_world_states, window_sizes = range(2,10)):
    '''
    Find all chunks for all structures (so we can search for the structures that involve this chunk)

    a 'window-size' is the amount of states between first and final one considered INCLUSIVE. i.e. n is n-1 actions.
    i.e. window size 3 means 2 consecutive actions
    '''

    df_target_grouped = df_proc_world_states.groupby(['gameID','targetName','phase_extended'])['flatDiscreteWorldStr']

    df_world_deltas = df_proc_trial.copy()

    for chunk_size in window_sizes:

        # for each reconstruction, get a list of ngrams of that length
        df_ngrams = df_target_grouped.agg(lambda ws: list(nltk.ngrams(list(ws), chunk_size))).reset_index()

        # find the chunks (world deltas) from those ngrams   
        df_ngrams['world_diff'] = df_ngrams['flatDiscreteWorldStr'].apply(lambda ngrams:
                                         ["".join([str(int(a)) for a in 
                                          list(
                                              np.logical_xor(np.array(list(ngram[-1])).astype(np.bool), 
                                              np.array(list(ngram[0])).astype(np.bool))
                                              )])
                                         for ngram in ngrams])

        df_ngrams = df_ngrams.rename(columns={"flatDiscreteWorldStr": str(chunk_size)+'_grams',
                                              "world_diff": str(chunk_size)+'_chunks'})

        df_world_deltas = df_world_deltas.merge(df_ngrams, how='left', on=['gameID','targetName','phase_extended'])
        
    # combine chunks from all window sized into list, so we can search for chunks in the entire reconstruction
    df_world_deltas['all_chunks'] = df_world_deltas[[(str(chunk_window)+'_chunks') \
                        for chunk_window in window_sizes if (str(chunk_window)+'_chunks') in df_world_deltas.columns]]\
    .apply(lambda row: [chunk for chunks in list(row) for chunk in chunks], axis=1)


    return df_world_deltas
def find_perc_chunks_in_procedures(perceptual_chunk_counters, 
                                   df_proc_chunks,
                                   min_count = 0):
    '''
    construct a new dataframe, searching for chunks 
    '''

    occurrences_in_procedures = pd.DataFrame()

    for target in targets:
        
        for i, (chunk_str, count) in enumerate(perceptual_chunk_counters[target].items()):

            if count >= min_count:

                props = {}

                for phase in ['pre','post']:
                    
                    subset_for_target = df_proc_chunks[(df_proc_chunks.blockFell == False) &
                                                       (df_proc_chunks.targetName == target) &
                                                       (df_proc_chunks.phase == phase)]

                    # check all precomputed chunks (of window sizes selected above) for presence of chunk
                    subset_with_chunk = subset_for_target[(subset_for_target['all_chunks']\
                                                           .apply(lambda chunks: chunk_str in chunks))]
                    
                    chunk_array = bc.chunk_str_to_cropped_array(chunk_str)
                    
                    
                    
                    if subset_with_chunk.shape[0]:
                        n_with_chunk = subset_with_chunk.shape[0]
                        props[phase] = subset_with_chunk.shape[0] / subset_for_target.shape[0]
                    else:
                        n_with_chunk = 0
                        props[phase] = 0

                    row = {
                        'targetName': target,
                        'phase': phase,
                        'chunk_str': chunk_str,
        #                 'reconstructions_with_chunk': list(subset_with_chunk['discreteWorld']),
                        'total_phase_reconstructions': subset_for_target.shape[0],
                        'n_with_chunk': n_with_chunk,
                        'chunk_height': np.sum(np.dot(np.sum(chunk_array, axis=0),np.arange(8)))/np.sum(chunk_array) + 0.5,
                        'proportion_with_chunk': props[phase],
                        'perceptual_count': count,
                        'perceptual_rank': i

                    }

                    occurrences_in_procedures = occurrences_in_procedures.append(row, ignore_index=True)

                occurrences_in_procedures.loc[(occurrences_in_procedures.targetName == target) & (occurrences_in_procedures.chunk_str == chunk_str), 'difference'] = props['post'] - props['pre']

                occurrences_in_procedures.loc[(occurrences_in_procedures.targetName == target) & (occurrences_in_procedures.chunk_str == chunk_str), 'both_zero'] = \
                    (props['pre'] == 0) & (props['post'] == 0)
                
    return occurrences_in_procedures

# for each structure, throw all chunks from all decompositions into a giant list
def addPerceptualChunks(chunk_list, decomposition, group_number):
    '''
    Checks whether a chunk with that group number exists in the decomposition and adds it to chunk_list 
    '''
    chunk = (decomposition==group_number)*1
    if chunk.any():
        chunk_list.append(bc.cropped_chunk_to_string(chunk))<jupyter_output><empty_output><jupyter_text>Properties of entire decompositions<jupyter_code># target = targets[0]

# a = np.zeros((len(ppts),len(ppts)))

# for i, ppt_i in enumerate(ppts):
#     for j, ppt_j in enumerate(ppts):
#         a[i,j] = decompositions[target][ppt_i] == decompositions[target][ppt_j]
# Convert each decomposition into a bag of chunks (for decomposition comparison)
# create dict of target to chunk list
decompositions = {}

for target in targets:

    decompositions[target] = {}
    
    for ppt in ppts:
        
        decompositions[target][ppt] = []
        
        for group in range(1,9):
            df_perc_trial[(df_perc_trial.targetName==target) & (df_perc_trial.gameID==ppt)].structureGrid.apply(\
                lambda decomposition: addPerceptualChunks(decompositions[target][ppt],
                                                                            decomposition,
                                                                            group))
            
        decompositions[target][ppt].sort()

# count and sort chunks
# perceptual_chunk_counters = {target: dict(Counter(perceptual_chunks[target]).most_common()) for target in targets}
from scipy import stats

def sem_CI(data, confidence=0.95, ddof = 0):
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), stats.sem(a, ddof = ddof)
    h = se * stats.t.ppf((1 + confidence) / 2., n-1)
    print( "$%.3g$ (%.3g\\%% CI: $[%.3g, %.3g]$)" % (m, confidence*100, m-h, m+h))
    return m, m-h, m+h

def bootstrap(niter, ppts, func):
    '''
    incomplete
    '''
    sample_indices = [random.randint(0,len(ppts)-1) for i in range(0, len(ppts))]
# number of unique decompositions for each target
all_decom = len(ppts) * len(targets)

unique_decom = [(len(np.unique(list(decompositions[target].values())))) for target in targets]
unique_decom

unique_decom_percent = [(len(np.unique(list(decompositions[target].values()))))/len(ppts) for target in targets]
sem_CI(unique_decom)
sem_CI(unique_decom_percent)
sum(unique) / all_decom<jupyter_output><empty_output><jupyter_text>Popular perceptual chunks for each structure  dataframe<jupyter_code>def chunkify_decomposition(df_perc_trial):
    
    df_chunk_properties = pd.DataFrame()
    
    for i, row in df_perc_trial.iterrows():
        for group_number in range(1,9):
            chunk = (row.structureGrid==group_number)*1
            if chunk.any():
                potential_block = bc.could_be_single_block(chunk)
                new_row = {
                    'chunk_str' : bc.cropped_chunk_to_string(chunk),
                    'chunk_array' : chunk,
                    'targetName': row.targetName,
                    'trialNum': row.trialNum,
                    'gameID': row.gameID,
                    'nChunksHighlighted': row.nChunksHighlighted,
                    'rectangular': np.bool(bc.is_rectangle(chunk)),
                    'is_block' : np.bool(potential_block[0]),
                    'which_block' : potential_block[1],
                    'height' : bc.chunk_height(chunk, chunk_is_string=False),
                    'area' : bc.chunk_area(chunk, chunk_is_string=False)
                    }
                df_chunk_properties = df_chunk_properties.append(new_row, ignore_index= True)
            
    return df_chunk_properties

# dataframe for properties of each chunk identifies (row for every time it was selected)
df_chunk_properties = chunkify_decomposition(df_perc_trial)

# dataframe for properties of unique chunks (one row per unique chunk in a structure)
df_chunk_properties_agg = df_chunk_properties.groupby(['targetName','chunk_str']).\
                                                        agg(count=('chunk_str', 'count'), 
                                                            height=('height', 'first'), 
                                                            area=('area', 'first'),
                                                            is_block=('is_block', 'first'),
                                                            rectangular=('rectangular','first'),
                                                            chunk_array=('chunk_array', 'first')).reset_index()
# mean of all chunks selected that are rectangular 
sem_CI(df_chunk_properties['rectangular'])
# mean of all chunks selected that are rectangular 
sem_CI(df_chunk_properties_agg['rectangular'])
# mean of all **unique chunks** that are rectangular (by structure)
sem_CI(1 - df_chunk_properties_agg.groupby('targetName').rectangular.mean())
# mean of all chunks selected that are rectangular 
sem_CI(df_chunk_properties_agg['is_block'])
plt.figure(figsize=(10,6))
sns.set_context('poster')
sns.set_style('whitegrid', {'legend':False})
sns.set(style="ticks", rc={"lines.linewidth": 1.5})
g = sns.FacetGrid(df_chunk_properties_agg, col="targetName", height=4)
g.map(sns.scatterplot, "count", "rectangular");
# number of times each chunk was built
df_chunk_properties_agg.groupby(['targetName','rectangular'])['count'].mean()
df_chunk_properties_agg.groupby(['targetName','rectangular'])['count'].mean().groupby('rectangular').mean()
df_chunk_properties_agg.groupby(['targetName','rectangular'])['count'].mean().groupby('rectangular').apply(sem_CI)
# looks like some ppts reliably use lots of rectangles

rect_prop = df_chunk_properties.groupby(['targetName','gameID']).rectangular.mean().groupby('gameID')

plt.bar(range(0,len(ppts)),height=rect_prop.mean(),yerr=rect_prop.std()/np.sqrt(8))
plt.hist(rect_prop.mean(), bins=12)
n_highlighted_chunks = df_perc_trial.groupby('targetName').nChunksHighlighted.mean()
sem_CI(n_highlighted_chunks)
np.std(n_highlighted_chunks, ddof=0)<jupyter_output><empty_output><jupyter_text>count dictionary<jupyter_code># Popular chunks
# create dict of target to chunk list
perceptual_chunks = {}

for target in targets:
    
    perceptual_chunks[target] = []
    
    for group in range(1,9):
        df_perc_trial[df_perc_trial.targetName==target].structureGrid.apply(\
            lambda decomposition: addPerceptualChunks(perceptual_chunks[target],
                                                                        decomposition,
                                                                        group))

# count and sort chunks
perceptual_chunk_counters = {target: dict(Counter(perceptual_chunks[target]).most_common()) for target in targets}
visualize = True

if visualize:

    n_chunks = 20

    fig, axs = plt.subplots(n_chunks, len(targets), figsize=(16,2*n_chunks))

    for i, target in enumerate(targets):
        for j in range(0, n_chunks):
            axs[j,i].axis('off')
            axs[j,i].set_title(str(list(perceptual_chunk_counters[target].values())[j]))
            # label as rectangular or not
            #chunk_str =  str(list(perceptual_chunk_counters[target].keys())[j])
            #axs[j,i].set_title(str(bc.is_rectangle(bc.chunk_str_to_cropped_array(chunk_str))))
            drawing.show_chunk([list(perceptual_chunk_counters[target].keys())[j]], axs[j,i], target=target, cmap='Blues', cropped=True)
            
visualize = True

if visualize:

    n_chunks = 8

    fig, axs = plt.subplots(len(targets), n_chunks, figsize=(16,2*n_chunks))

    for i, target in enumerate(targets):
        for j in range(0, n_chunks):
            axs[i,j].axis('off')
            axs[i,j].set_title(str(list(perceptual_chunk_counters[target].values())[j]))
            # label as rectangular or not
            #chunk_str =  str(list(perceptual_chunk_counters[target].keys())[j])
            #axs[j,i].set_title(str(bc.is_rectangle(bc.chunk_str_to_cropped_array(chunk_str))))
            drawing.show_chunk([list(perceptual_chunk_counters[target].keys())[j]], axs[i,j], target=target, cmap='Blues', cropped=True)

            
# plot settings for paper
a = sns.light_palette("#1C373E", reverse=False, as_cmap=True)
sns.set(font_scale=1.5)
sns.set_style('whitegrid', {'font.family':'sans-serif', 'font.sans-serif':'Avenir'})
subset = targets[4:8]
visualize = True

sns.set(font_scale=1.5)
sns.set_style('whitegrid', {'font.family':'sans-serif', 'font.sans-serif':'Avenir'})

if visualize:

    n_chunks = 8

    fig, axs = plt.subplots(len(targets), n_chunks, figsize=(16,2*n_chunks))

    for i, target in enumerate(targets):
        for j in range(0, n_chunks):
            axs[i,j].axis('off')
            axs[i,j].set_title(str(list(perceptual_chunk_counters[target].values())[j]))
            # label as rectangular or not
            #chunk_str =  str(list(perceptual_chunk_counters[target].keys())[j])
            #axs[j,i].set_title(str(bc.is_rectangle(bc.chunk_str_to_cropped_array(chunk_str))))
            drawing.show_chunk([list(perceptual_chunk_counters[target].keys())[j]], axs[i,j], target=target, cmap=a, cropped=True)
            
sns.set(font_scale=1.5)
sns.set_style('whitegrid', {'font.family':'sans-serif', 'font.sans-serif':'Avenir'})

subset = targets[4:8]

if visualize:

    n_chunks = 8

    fig, axs = plt.subplots(n_chunks, len(subset), figsize=(len(subset)*2,2*n_chunks))

    for i, target in enumerate(subset):
        for j in range(0, n_chunks):
            axs[j,i].axis('off')
            axs[j,i].set_title(str(list(perceptual_chunk_counters[target].values())[j]))
            # label as rectangular or not
            #chunk_str =  str(list(perceptual_chunk_counters[target].keys())[j])
            #axs[j,i].set_title(str(bc.is_rectangle(bc.chunk_str_to_cropped_array(chunk_str))))
            drawing.show_chunk([list(perceptual_chunk_counters[target].keys())[j]], axs[j,i], target=target, cmap=a, cropped=True)
            
fig.savefig('test_gallery.pdf', transparent=True, bbox_inches='tight', pad_inches=0)
# pickle.dump(perceptual_chunk_counters, open("perceptual_chunk_counters.p", "wb" ))
# counts = list(perceptual_chunk_counters[target].values())
# plt.bar(x=range(0,len(counts)), height=counts, bins=len(counts))
# # plt.hist(counts, bins=len(counts))
plt.figure(figsize=(10,6))
sns.set_context('poster')
sns.set_style('whitegrid', {'legend':False})
sns.set(style="ticks", rc={"lines.linewidth": 1.5})
for target in targets:
    plt.plot(list(perceptual_chunk_counters[target].values()))<jupyter_output><empty_output><jupyter_text>Find popular chunks<jupyter_code>#### Number of unique chunks per structure
n_unique_chunks_all = {target : len(np.array(list(perceptual_chunk_counters[target].keys()))) for target in targets}
n_unique_chunks_all
#### Total unique chunks
total_unique_chunks_identified = sum(n_unique_chunks_all.values())
total_unique_chunks_identified
prop_chunks_to_include = 0.85
# If choosing chunks by proportion of unique chunks

# prop_chunks_to_include = 0.90

# k = 0

# while sum([len(np.array(list(perceptual_chunk_counters[target].keys()))[0:k]) for target in targets]) < total_unique_chunks_identified*prop_chunks_to_include:
#     k+=1
    
# k
# If choosing chunks by proportion of all chunks selected (unique or not)

total_chunks_identified = sum([np.sum(np.array(list(perceptual_chunk_counters[target].values()))) for target in targets])
total_chunks_identified
k = 0

while sum([np.sum(np.array(list(perceptual_chunk_counters[target].values()))[0:k]) for target in targets]) < total_chunks_identified*prop_chunks_to_include:
    k+=1
    
print('top ' + str(k) + ' chunks selected from each tower to cover ' + str(prop_chunks_to_include*100) + '% of all chunks')


means = []

for target in targets:
    counts = list(perceptual_chunk_counters[target].values())
#     total_chunks = sum(vals)
    means.append(np.mean(counts))

means<jupyter_output><empty_output><jupyter_text>Load in building procedures from block_silhouette, and find all world-deltas for all reconstructions'world-deltas': change in world state (i.e. squares covered by blocks) between action i and action j, for all i and j.<jupyter_code># load in procedural data from silhouette experiment
silhouette_world_path = os.path.join(silhouette_csv_dir,'procedural_chunks_world_states_{}.p'.format('Exp2Pilot3_all'))
df_proc_world_states = pickle.load(open(silhouette_world_path, "rb" ))


silhouette_trial_path = os.path.join(silhouette_csv_dir,'block_silhouette_{}_good.csv'.format('Exp2Pilot3_all'))
df_proc_trial = pd.read_csv(silhouette_trial_path)
# find world-deltas in procedures
df_proc_chunks = find_world_diffs(df_proc_world_states)
# search for perceptual chunks in world-deltas for each phase
df_occurrences_in_procedures = find_perc_chunks_in_procedures(perceptual_chunk_counters, 
                                                                df_proc_chunks,
                                                                min_count = 0)
# main dataframe for perceptual chunk stats, by phase

properties_to_copy = df_chunk_properties.groupby(['chunk_str','targetName']).first()\
                .reset_index()[['chunk_str','targetName','height','area','rectangular','is_block',
                                                 'which_block']]

df_perceptual_chunks_by_phase = df_occurrences_in_procedures.merge(properties_to_copy, 
                                                          how='left', 
                                                          on=['chunk_str','targetName'], )



out_path = os.path.join(csv_dir,'df_perceptual_chunks_by_phase.csv')
df_perceptual_chunks_by_phase.to_csv(out_path)
# main dataframe for perceptual chunk stats, with both phases aggregated

df_perceptual_chunks = df_perceptual_chunks_by_phase.groupby(['chunk_str','targetName',
                                                             'perceptual_count']).agg(\
                                                n_with_chunk = ('n_with_chunk', 'sum'),
                                                total_reconstructions = ('total_phase_reconstructions','sum'),
                                                area = ('area','first'),
                                                height = ('height','first'),
                                                rectangular = ('rectangular','first'),
                                                both_zero = ('both_zero','first'),
                                                is_block = ('is_block','first'),
                                                which_block = ('which_block','first'),
                                                perceptual_rank = ('perceptual_rank','first')).reset_index()

df_perceptual_chunks['proportion_with_chunk'] = df_perceptual_chunks['n_with_chunk']/\
                                                df_perceptual_chunks['total_reconstructions']
out_path = os.path.join(csv_dir,'df_perceptual_chunks.csv')
df_perceptual_chunks.to_csv(out_path)
# out_path = os.path.join(csv_dir,'df_perceptual_chunks.csv')
# df_perceptual_chunks.to_csv(out_path)<jupyter_output><empty_output><jupyter_text>Chunks with particular properties (overall- not separated by phase)<jupyter_code># filter and sort by most frequently built (in both phases) 
built_top_perceptual = df_perceptual_chunks[(df_perceptual_chunks.n_with_chunk > 0) &
                                             (df_perceptual_chunks.perceptual_rank < k)]\
                            .sort_values(['n_with_chunk'],ascending=False).reset_index()<jupyter_output><empty_output><jupyter_text>perceptual chunks that occur most frequently in building procedures<jupyter_code>n_to_view = 20

fig, axs = plt.subplots(1, n_to_view, figsize=(20,2.5*n_to_view))

for i in range(0,n_to_view):
    axs[i].axis('off')
    axs[i].set_title(str(int(built_top_perceptual.loc[i,:].n_with_chunk)))
    drawing.show_chunk([built_top_perceptual.loc[i,:].chunk_str], axs[i], target=built_top_perceptual.loc[i,:].targetName, cmap='Blues', cropped=True)<jupyter_output><empty_output><jupyter_text>non-rectangular perceptual chunks that occur in building procedures<jupyter_code># find non-rectangular perceptual chunks that wre built at all
# non_rect_chunks = df_perceptual_chunks[(df_perceptual_chunks.n_with_chunk > 0) &
#                      (df_perceptual_chunks.rectangular == 0.)&
#                      (df_perceptual_chunks.perceptual_rank < k)].reset_index().sort_values(['n_with_chunk'])

non_rect_chunks = built_top_perceptual[(built_top_perceptual.rectangular == 0.)].reset_index().sort_values(['n_with_chunk'])


fig, axs = plt.subplots(1, len(non_rect_chunks), figsize=(20,2.5*len(non_rect_chunks)))

for i, row in non_rect_chunks.iterrows():
    axs[i].axis('off')
    axs[i].set_title(str(int(row.n_with_chunk)))
    drawing.show_chunk([row.chunk_str], axs[i], target=row.targetName, cmap='Blues', cropped=True)<jupyter_output><empty_output><jupyter_text>(non) block-placement perceptual chunks that occur in building procedures<jupyter_code># Only look at ones that aren't block placements
# non-rectangular perceptual chunks that occur in building procedures

# filter and sort by most frequently built (in both phases) 
built_top_perceptual_not_block = built_top_perceptual[(built_top_perceptual.is_block == False)]\
                        .sort_values(['n_with_chunk'],ascending=False).reset_index()

n_to_view = 20

fig, axs = plt.subplots(1, n_to_view, figsize=(20,2.5*n_to_view))

for i in range(0,n_to_view):
    axs[i].axis('off')
    axs[i].set_title(str(int(built_top_perceptual_not_block.loc[i,:].n_with_chunk)))
    drawing.show_chunk([built_top_perceptual_not_block.loc[i,:].chunk_str], axs[i],
                       target=built_top_perceptual_not_block.loc[i,:].targetName, cmap='Blues', cropped=True)
not_single_block = len(built_top_perceptual[built_top_perceptual.is_block==1])
single_block = len(built_top_perceptual[built_top_perceptual.is_block==0])

single_block/(not_single_block+single_block)<jupyter_output><empty_output><jupyter_text>Chunk appearance in procedures separated by phase<jupyter_code>df_top_k_phase = df_perceptual_chunks_by_phase[df_perceptual_chunks_by_phase.perceptual_rank < k]

# # by number of times built
# min_occurrences = 3
# df_min_occurrences_phase = df_perceptual_chunks_by_phase[df_perceptual_chunks_by_phase.perceptual_count >= min_occurrences]
# print(len(df_min_occurrences_phase))
# Were perceptual chunks built more in the first or final repetition? By structure

fig = plt.figure(figsize=(10,6))
sns.set_context('poster')
sns.set_style('whitegrid', {'legend':False})
sns.set(style="ticks", rc={"lines.linewidth": 0.7})
sns.scatterplot(data=df_top_k_phase, x='n_with_chunk', y='perceptual_count', hue='rectangular')
df_top_k_phase_built = df_top_k_phase[df_top_k_phase.both_zero==False]
# Were perceptual chunks built more in the first or final repetition? By structure

fig = plt.figure(figsize=(10,6))
sns.set_context('poster')
sns.set_style('whitegrid', {'legend':False})
sns.set(style="ticks", rc={"lines.linewidth": 0.7})
sns.pointplot(data=df_top_k_phase_built, x='phase', y='proportion_with_chunk', hue='targetName')
# How many chunks were build more, and how many were built less?

fig = plt.figure(figsize=(14,10))
sns.set_context('poster')
sns.set_style('whitegrid', {'legend':False})
sns.set(style="ticks", rc={"lines.linewidth": 0.7})
g = sns.FacetGrid(data=df_top_k_phase_built, col="targetName", hue="chunk_str", col_order=targets)
g.map(sns.pointplot,"phase","proportion_with_chunk", order=['pre','post'])<jupyter_output><empty_output><jupyter_text>Visualize top movers in either direction<jupyter_code>df = df_top_k_phase_built[(df_top_k_phase_built.phase=='pre')]

n_chunks = 6

fig, axs = plt.subplots(len(targets), n_chunks*2, figsize=(4*n_chunks,2.5*len(targets)))

for i, target in enumerate(targets):
    for j in range(0, n_chunks):    
        greatest_increase = df[df.targetName==target].sort_values('difference', ascending=False).reset_index()
        axs[i,j].axis('off')
        axs[i,j].set_title(str((round(greatest_increase.loc[j,'difference'],4))))
        drawing.show_chunk([greatest_increase.loc[j,'chunk_str']], axs[i,j], target=target, cmap='Blues', cropped=True)

for i, target in enumerate(targets):
    for j in range(0, n_chunks):    
        greatest_decrease = df[df.targetName==target].sort_values('difference', ascending=True).reset_index()
        axs[i,n_chunks*2-1-j].axis('off')
        axs[i,n_chunks*2-1-j].set_title(str((round(greatest_decrease.loc[j,'difference'],4))))
        drawing.show_chunk([greatest_decrease.loc[j,'chunk_str']], axs[i,n_chunks*2-1-j], target=target, cmap='Oranges', cropped=True)

# <-- Largest increase first to final ... Largest decrease first to final--><jupyter_output><empty_output><jupyter_text>Unbuilt popular perceptual chunks<jupyter_code>unbuilt_popular_perceptual = df_perceptual_chunks[(df_perceptual_chunks.n_with_chunk==0.) & 
                                                 (df_perceptual_chunks.perceptual_rank < k)]\
                                .sort_values('perceptual_count', ascending=False).reset_index()


n_top_unbuilt = 20

fig, axs = plt.subplots(1, n_top_unbuilt, figsize=(20,2.5*n_top_unbuilt))

for i in range(0,n_top_unbuilt):
    axs[i].axis('off')
    axs[i].set_title(str(int(unbuilt_popular_perceptual.loc[i,:].perceptual_count)))
    drawing.show_chunk([unbuilt_popular_perceptual.loc[i,:].chunk_str], 
                       axs[i], target=unbuilt_popular_perceptual.loc[i,:].targetName, cmap='Blues', cropped=True)
built_popular_perceptual = df_perceptual_chunks[(df_perceptual_chunks.n_with_chunk>0.) & 
                                                 (df_perceptual_chunks.perceptual_rank<k)]
prop_unbuilt = len(unbuilt_popular_perceptual)/(len(built_popular_perceptual) + len(unbuilt_popular_perceptual))
prop_built = len(built_popular_perceptual)/(len(built_popular_perceptual) + len(unbuilt_popular_perceptual))
prop_built
prop_unbuilt
# percent not built at all
df_perceptual_chunks[df_perceptual_chunks.perceptual_rank < k].groupby('targetName')['both_zero'].mean()
# CI on unbuilt perceptual chunks
sem_CI(100*df_perceptual_chunks[df_perceptual_chunks.perceptual_rank < k].groupby('targetName')['both_zero'].mean())
sem_CI(100 - 100*df_perceptual_chunks[df_perceptual_chunks.perceptual_rank < k].groupby('targetName')['both_zero'].mean())
# Most popular perceptual chunks in each phase

n_chunks = 5

fig, axs = plt.subplots(len(targets), n_chunks*2, figsize=(4*n_chunks,2.5*len(targets)))

for i, target in enumerate(targets):
    for j in range(0, n_chunks):    
        most_popular = df_top_k_phase[(df_top_k_phase.targetName==target) &
                                                       (df_top_k_phase.phase=='pre')].sort_values('n_with_chunk', ascending=False).reset_index()
        axs[i,j].set_title(str(round(most_popular.loc[j,'n_with_chunk'],2)))
        axs[i,j].axis('off')
        drawing.show_chunk([most_popular.loc[j,'chunk_str']], axs[i,j], target=target, cmap='Blues', cropped=True)

for i, target in enumerate(targets):
    for j in range(0, n_chunks):    
        most_popular = df_top_k_phase[(df_top_k_phase.targetName==target) &
                                                       (df_top_k_phase.phase=='post')].sort_values('n_with_chunk', ascending=False).reset_index()
#         axs[i,n_chunks*2-1-j].set_title(str(round(most_popular.loc[j,'built_by_ppt'],2)))
#         axs[i,n_chunks*2-1-j].axis('off')
#         drawing.show_chunk([most_popular.loc[j,'chunk']], axs[i,n_chunks*2-1-j], target=target, cmap='Reds', cropped=True)
        axs[i,int(j+n_chunks)].set_title(str(round(most_popular.loc[j,'n_with_chunk'],2)))
        axs[i,int(j+n_chunks)].axis('off')
        drawing.show_chunk([most_popular.loc[j,'chunk_str']], axs[i,int(j+n_chunks)], target=target, cmap='Oranges', cropped=True)

# Most popular perceptual chunks in each phase

n_chunks = 10

fig, axs = plt.subplots(len(targets), n_chunks, figsize=(2*n_chunks,2.5*len(targets)))

for i, target in enumerate(targets):
    for j in range(0, n_chunks):    
        most_popular = built_top_perceptual[(built_top_perceptual.targetName==target)]\
            .sort_values('n_with_chunk', ascending=False).reset_index()
        axs[i,j].set_title(str(round(most_popular.loc[j,'n_with_chunk'],2)))
        axs[i,j].axis('off')
        drawing.show_chunk([most_popular.loc[j,'chunk_str']], axs[i,j], target=target, cmap='Blues', cropped=True)
# How many chunks were built more, and how many were built less?

fig = plt.figure(figsize=(20,10))
sns.set_context('poster')
sns.set_style('whitegrid', {'legend':False})
sns.set(style="ticks", rc={"lines.linewidth": 4})
g = sns.FacetGrid(data=df_top_k_phase_built, col="targetName", col_wrap=4, hue="chunk_str", col_order=targets)
g.map(sns.lineplot, "perceptual_rank", "proportion_with_chunk")
g.map(sns.scatterplot, "perceptual_rank", "proportion_with_chunk")


# How many chunks were build more, and how many were built less?

df = df_occurrences_in_procedures[(df_occurrences_in_procedures.perceptual_count > 2)]

fig = plt.figure(figsize=(20,10))
sns.set_context('poster')
sns.set_style('whitegrid', {'legend':True})
sns.set(style="ticks", rc={"lines.linewidth": 4})
g = sns.FacetGrid(data=df, col="targetName", col_wrap=4, hue="phase", col_order=targets)
g.map(sns.scatterplot, "perceptual_count", "proportion_with_chunk")
plt.legend()

target = 'hand_selected_004'
c1 = df_occurrences_in_procedures[(df_occurrences_in_procedures.targetName == target) &
                             (df_occurrences_in_procedures.proportion_with_chunk > 0.6)].chunk_str.iloc[0]
drawing.show_chunk([c1], plt, target=target, cmap='Blues', cropped=True)
target = 'hand_selected_016'
c2 = df_occurrences_in_procedures[(df_occurrences_in_procedures.targetName == target)&
                             (df_occurrences_in_procedures.perceptual_count > 30)].chunk_str.iloc[3]
drawing.show_chunk([c2], plt, target=target, cmap='Blues', cropped=True)
df_occurrences_in_procedures
df_occurrences_in_procedures
out_path = os.path.join(csv_dir,'df_perceptual_in_procedures.csv')
df_occurrences_in_procedures.to_csv(out_path)
drawing.show_chunk([c], plt, target='hand_selected_005', cmap='Blues', cropped=True)
c = list(perceptual_chunk_counters['hand_selected_005'].keys())[0]<jupyter_output><empty_output><jupyter_text>Integrate chunk information and perceptual rank and count into dataframe<jupyter_code>df_chunk_use_separate_phases = pickle.load(open("df_chunk_use_separate_phases_2to6.p", "rb" ))
df_chunk_use_separate_phases['chunk_str'] = df_chunk_use_separate_phases['chunk']
df_chunk_use_separate_phases['chunk_array'] = \
            df_chunk_use_separate_phases['chunk'].apply(bc.chunk_str_to_cropped_array)

df_chunk_use_separate_phases['rectangular'] = df_chunk_use_separate_phases['chunk_array'].apply(lambda chunk: np.bool(bc.is_rectangle(chunk)))
df_chunk_use_separate_phases['is_block'] = df_chunk_use_separate_phases['chunk_array'].apply(lambda chunk: np.bool(bc.could_be_single_block(chunk)[0]))
df_chunk_use_separate_phases['which_block'] = df_chunk_use_separate_phases['chunk_array'].apply(lambda chunk: bc.could_be_single_block(chunk)[1])
df_chunk_use_separate_phases['height'] = df_chunk_use_separate_phases['chunk_array'].apply(lambda chunk: bc.chunk_height(chunk, chunk_is_string=False))
df_chunk_use_separate_phases['area'] = df_chunk_use_separate_phases['chunk_array'].apply(lambda chunk: bc.chunk_area(chunk, chunk_is_string=False))
df_chunk_use_separate_phases
proc_chunks_with_perc_data = \
    df_chunk_use_separate_phases.merge(\
            df_perceptual_chunks[['chunk_str','targetName','perceptual_count','perceptual_rank']],\
                                       how='left', on=['chunk_str','targetName'])

proc_chunks_with_perc_data['is_top_perceptual_chunk'] = proc_chunks_with_perc_data['perceptual_rank'] < k

out_path = os.path.join(csv_dir,'proc_chunks_with_perc_data.csv')
proc_chunks_with_perc_data.to_csv(out_path)

df = proc_chunks_with_perc_data
# How often were most popular chunks built by left-out ppt, by phase, by structure?

fig = plt.figure(figsize=(10,6))
sns.set_context('poster')
sns.set_style('whitegrid', {'legend':False})
sns.set(style="ticks", rc={"lines.linewidth": 0.7})
g = sns.FacetGrid(df, col="chunk_window", height=4, hue='is_top_perceptual_chunk')
g.map(sns.pointplot, "phase", "built_by_ppt", order=['pre','post']);<jupyter_output><empty_output><jupyter_text>Set analysis<jupyter_code>top_perc = df_perceptual_chunks[df_perceptual_chunks.perceptual_rank<k]\
                .groupby(['targetName','chunk_str']).first().groupby(['targetName']).size()
top_perc
built_perc = proc_chunks_with_perc_data[proc_chunks_with_perc_data.is_top_perceptual_chunk]\
    .groupby(['targetName','chunk']).first().groupby(['targetName']).size()
built_perc
all_proc = proc_chunks_with_perc_data.groupby(['targetName','chunk']).first().groupby(['targetName']).size()
built_perc
sem_CI(built_perc/top_perc)
sem_CI(built_perc/all_proc)
proc_chunks_with_perc_data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12627428070687893, 'fraction_numerical': 0.007974295311815394, 'mean_word_length': 3.4782210595160237, 'pattern_counts': {'":': 10, '<': 74, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 89, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '632861', 'license': 'MIT', 'n_tokens_mistral': 11636, 'n_tokens_neox': 10987, 'n_words': 1752, 'path': 'analysis/cogsci2021/.ipynb_checkpoints/perceptual_chunks-checkpoint.ipynb', 'repo_name': 'jacobschenberg/block_construction', 'size': '34235'}