<jupyter_start><jupyter_text>analyse<jupyter_code>import numpy as np

def show_values_on_bars(axs):
    def _show_on_single_plot(ax):        
        for p in ax.patches:
            _x = p.get_x() + p.get_width() / 2
            _y = p.get_y() + p.get_height() + 0.02
            value = '{:.2f}'.format(p.get_height())
            ax.text(_x, _y, value, ha="center") 

    if isinstance(axs, np.ndarray):
        for idx, ax in np.ndenumerate(axs):
            _show_on_single_plot(ax)
    else:
        _show_on_single_plot(axs)

pal = cols
data = joined.loc[joined['cluster'].isin()]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)

sns.set(context="paper", style="ticks", rc={'patch.force_edgecolor': False})
fig, ax = plt.subplots(figsize=(10, 5))
sns.barplot(ax=ax, x=data.index, y=data, order=data.index, palette=pal)
sns.despine(offset=10)
plt.ylabel('frequency')
plt.xlabel('historical period')
plt.ylim(0, 1)
show_values_on_bars(ax)
sample = joined.loc[joined['STRUKTURA_STAV'].isin([1, 2, 5, 6, 7, 8, 9])]
data = sample.loc[sample['cluster'].isin([11])]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)

labels = ['organic', 'perimeter block', 'village', 'garden city', 'modernism', 'production', 'services']
sns.set(context="paper", style="ticks", rc={'patch.force_edgecolor': False})
fig, ax = plt.subplots(figsize=(10, 5))
sns.barplot(ax=ax, x=data.index, y=data, order=[1, 2, 5, 6, 7, 8, 9], palette=pal)
sns.despine(offset=10)
plt.ylabel('frequency')
plt.xlabel('qualitative typology')
plt.ylim(0, 1)
ax.set_xticklabels(labels)
show_values_on_bars(ax)
# save all clusters
for cl in range(20):
    data = sample.loc[sample['cluster'].isin([cl])]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)

    fig, ax = plt.subplots(figsize=(10, 5))
    sns.barplot(ax=ax, x=data.index, y=data, order=[1, 2, 5, 6, 7, 8, 9], palette=pal)
    sns.despine(offset=10)
    plt.ylabel('frequency')
    plt.xlabel('qualitative typology')
    plt.ylim(0, 1)
    ax.set_xticklabels(labels)
    show_values_on_bars(ax)
    for ext in ['pdf', 'png']:
        plt.savefig('figures/PRG_cluster_' + str(cl) + '_structure.' + ext, bbox_inches='tight')
    plt.close()
fig, ax = plt.subplots(2, 2, figsize=(14, 10))

data = sample.loc[sample['cluster'].isin([11])]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)

sns.barplot(ax=ax[0, 0], x=data.index, y=data, order=[1, 2, 5, 6, 7, 8, 9], palette=pal)
sns.despine(offset=10)
ax[0,0].set_ylabel('frequency')
ax[0,0].set_xlabel('qualitative typology')
ax[0,0].set_title('cluster 11')
ax[0,0].set_ylim(0, 1)
ax[0,0].set_xticklabels(labels)
show_values_on_bars(ax[0, 0])

data = sample.loc[sample['cluster'].isin([5])]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)
sns.barplot(ax=ax[0, 1], x=data.index, y=data, order=[1, 2, 5, 6, 7, 8, 9], palette=pal)
sns.despine(offset=10)
ax[0,1].set_ylabel('frequency')
ax[0,1].set_xlabel('qualitative typology')
ax[0,1].set_title('cluster 5')
ax[0,1].set_ylim(0, 1)
ax[0,1].set_xticklabels(labels)
show_values_on_bars(ax[0, 1])

data = sample.loc[sample['cluster'].isin([12])]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)
sns.barplot(ax=ax[1, 0], x=data.index, y=data, order=[1, 2, 5, 6, 7, 8, 9], palette=pal)
sns.despine(offset=10)
ax[1,0].set_ylabel('frequency')
ax[1,0].set_xlabel('qualitative typology')
ax[1,0].set_title('cluster 12')
ax[1,0].set_ylim(0, 1)
ax[1,0].set_xticklabels(labels)
show_values_on_bars(ax[1, 0])

data = sample.loc[sample['cluster'].isin([13])]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)
sns.barplot(ax=ax[1, 1], x=data.index, y=data, order=[1, 2, 5, 6, 7, 8, 9], palette=pal)
sns.despine(offset=10)
ax[1,1].set_ylabel('frequency')
ax[1,1].set_xlabel('qualitative typology')
ax[1,1].set_title('cluster 13')
ax[1,1].set_ylim(0, 1)
ax[1,1].set_xticklabels(labels)
show_values_on_bars(ax[1, 1])

plt.tight_layout()
plt.savefig('figures/PRG_cluster_structure_subplot.pdf')
fig, ax = plt.subplots(2, 2, figsize=(14, 10))

data = sample.loc[sample['cluster'].isin([11, 15, 5])]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)

sns.barplot(ax=ax[0, 0], x=data.index, y=data, order=[1, 2, 5, 6, 7, 8, 9], palette=pal)
sns.despine(offset=10)
ax[0,0].set_ylabel('frequency')
ax[0,0].set_xlabel('qualitative typology')
ax[0,0].set_title('compact city')
ax[0,0].set_ylim(0, 1)
ax[0,0].set_xticklabels(labels)
show_values_on_bars(ax[0, 0])

data = sample.loc[sample['cluster'].isin([3, 0, 8, 9, 13, 17])]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)
sns.barplot(ax=ax[0, 1], x=data.index, y=data, order=[1, 2, 5, 6, 7, 8, 9], palette=pal)
sns.despine(offset=10)
ax[0,1].set_ylabel('frequency')
ax[0,1].set_xlabel('qualitative typology')
ax[0,1].set_title('low-rise city')
ax[0,1].set_ylim(0, 1)
ax[0,1].set_xticklabels(labels)
show_values_on_bars(ax[0, 1])

data = sample.loc[sample['cluster'].isin([1, 19])]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)
sns.barplot(ax=ax[1, 0], x=data.index, y=data, order=[1, 2, 5, 6, 7, 8, 9], palette=pal)
sns.despine(offset=10)
ax[1,0].set_ylabel('frequency')
ax[1,0].set_xlabel('qualitative typology')
ax[1,0].set_title('industrial city')
ax[1,0].set_ylim(0, 1)
ax[1,0].set_xticklabels(labels)
show_values_on_bars(ax[1, 0])

data = sample.loc[sample['cluster'].isin([12, 14, 2, 10])]['STRUKTURA_STAV'].value_counts(sort=False, normalize=True)
sns.barplot(ax=ax[1, 1], x=data.index, y=data, order=[1, 2, 5, 6, 7, 8, 9], palette=pal)
sns.despine(offset=10)
ax[1,1].set_ylabel('frequency')
ax[1,1].set_xlabel('qualitative typology')
ax[1,1].set_title('heterogenous dense city branch')
ax[1,1].set_ylim(0, 1)
ax[1,1].set_xticklabels(labels)
show_values_on_bars(ax[1, 1])

plt.tight_layout()
plt.savefig('figures/PRG_branch_structure_subplot.pdf')
import scipy.stats as ss
import numpy as np

def cramers_v(x, y):
    confusion_matrix = pd.crosstab(x,y)
    chi2 = ss.chi2_contingency(confusion_matrix)[0]
    n = confusion_matrix.sum().sum()
    phi2 = chi2/n
    r,k = confusion_matrix.shape
    phi2corr = max(0, phi2-((k-1)*(r-1))/(n-1))
    rcorr = r-((r-1)**2)/(n-1)
    kcorr = k-((k-1)**2)/(n-1)
    return np.sqrt(phi2corr/min((kcorr-1),(rcorr-1)))
cramers_v(sample.cluster, sample.STRUKTURA_STAV)
confusion_matrix = pd.crosstab(sample.cluster, sample.STRUKTURA_STAV)
chi, p, dof, exp = ss.chi2_contingency(confusion_matrix)
p
dof
chi<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20295719844357976, 'fraction_numerical': 0.049961089494163426, 'mean_word_length': 6.377726750861079, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '20099', 'license': 'MIT', 'n_tokens_mistral': 2990, 'n_tokens_neox': 2753, 'n_words': 434, 'path': 'source/notebooks/chapter78/Chapter 8 - Comparison of taxa and qualitative classification.ipynb', 'repo_name': 'martinfleis/phd-thesis', 'size': '6425'}