<jupyter_start><jupyter_text>Measure simpson as patterned character<jupyter_code>import geopandas as gpd
import momepy as mm
from tqdm import tqdm
import numpy as np
import pandas as pd
import libpysal
import scipy as sp
import matplotlib.pyplot as plt
import seaborn as sns
import mapclassify
gdf = pd.read_csv('/Users/martin/Dropbox/Academia/Data/Geo/Prague/Clustering/primary.csv', index_col=0)
spatial_weights = libpysal.io.open('/Users/martin/Dropbox/Academia/Data/Geo/Prague/Clustering/GRqueen3.gal', 'r').read()
spatial_weights.neighbors = {int(k): [int(i) for i in v] for k, v in spatial_weights.neighbors.items()}
gdf.set_index('uID', inplace=True)
chars = gdf.columns
skewness = pd.DataFrame(index=chars)
for c in chars:
    skewness.loc[c, 'skewness'] =sp.stats.skew(gdf[c])
headtail = list(skewness.loc[skewness.skewness >= 1].index)<jupyter_output><empty_output><jupyter_text>some values will need to be inverted to follow heavy-tailed distribution<jupyter_code>to_invert = skewness.loc[skewness.skewness <= -1].index
for inv in to_invert:
    gdf[inv + '_r'] = gdf[inv].max() - gdf[inv]
inverted = [x for x in gdf.columns if '_r' in x]
headtail = headtail + inverted
natural = [x for x in chars if x not in headtail]<jupyter_output><empty_output><jupyter_text>CALCULATE<jupyter_code>def _simpson_di(data):

    """ Given a hash { 'species': count } , returns the Simpson Diversity Index

    >>> simpson_di({'a': 10, 'b': 20, 'c': 30,})
    0.3888888888888889

    https://gist.github.com/martinjc/f227b447791df8c90568
    """

    def p(n, N):
        """ Relative abundance """
        if n == 0:
            return 0
        return float(n) / N

    N = sum(data.values())

    return sum(p(n, N) ** 2 for n in data.values() if n != 0)
import mapclassify.classifiers as classifiers
schemes = {}
for classifier in classifiers.CLASSIFIERS:
    schemes[classifier.lower()] = getattr(classifiers, classifier)
results = {}
for c in headtail + natural:
    results[c] = []
bins = {}
for c in headtail:
    bins[c] = schemes['headtailbreaks'](gdf[c]).bins
for c in natural:
    bins[c] = mapclassify.gadf(gdf[c], method='NaturalBreaks')[1].bins
bins
for index, row in tqdm(gdf.iterrows(), total=gdf.shape[0]):
    neighbours = spatial_weights.neighbors[index].copy()
    neighbours.append(index)
    
    subset = gdf.loc[neighbours]
    for c in headtail + natural:
        values = subset[c]
        sample_bins = classifiers.UserDefined(values, list(bins[c]))
        counts = dict(zip(bins[c], sample_bins.counts))
        results[c].append(_simpson_di(counts))
for c in headtail + natural:
    gdf[c + '_simpson'] = results[c]
gdf
sim = [x for x in gdf.columns if '_simpson' in x]
simpson = gdf[sim + ['uID']]
simpson.rename(columns={'sscERI_r_simpson': 'sscERI_simpson',
 'ssbCCo_r_simpson': 'ssbCCo_simpson',
 'ssbERI_r_simpson': 'ssbERI_simpson',
 'mtdDeg_r_simpson': 'mtdDeg_simpson',
 'sssLin_r_simpson': 'sssLin_simpson',
 'lskERI_r_simpson': 'lskERI_simpson',}, inplace=True)
simpson.to_csv('/Users/martin/Dropbox/Academia/Data/Geo/Prague/Clustering/simpson.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13804173354735153, 'fraction_numerical': 0.015409309791332263, 'mean_word_length': 4.624548736462094, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '271647', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 1252, 'n_tokens_neox': 1184, 'n_words': 258, 'path': 'code_production/Prague/06Simpson_diversity.ipynb', 'repo_name': 'martinfleis/numerical-taxonomy-paper', 'size': '3115'}