<jupyter_start><jupyter_text>Persistent homology examples* Ripser [paper](https://www.theoj.org/joss-papers/joss.00925/10.21105.joss.00925.pdf) [code](https://github.com/scikit-tda/ripser.py) (fast)* Dionysus 2 [code](https://mrzv.org/software/dionysus2/) (representative examples)* Nico's [code](https://github.com/nhchristianson/Math-text-semantic-networks)* Ann's [code](https://github.com/asizemore/PH_tutorial/blob/master/Tutorial_day1.ipynb)<jupyter_code>%load_ext autoreload
%autoreload 2
import os,sys
sys.path.insert(1, os.path.join(sys.path[0], '..', 'module'))<jupyter_output><empty_output><jupyter_text>Try persistent homology<jupyter_code>import wiki

# topic = 'biochemistry'
topic = 'physics'
path_saved = '/Users/harangju/Box Sync/Research/my papers/wikipedia paper/data/graphs/full/'
network = wiki.Net()
network.load_graph(path_saved + topic + '.gexf')
%time network.barcodes
network.save_barcodes(path_saved + topic + '.p')
network.load_barcodes(path_saved + topic + '.p')<jupyter_output><empty_output><jupyter_text>Plot<jupyter_code>import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib._color_data as mcd

sns.set(style='whitegrid', font_scale=1.6)
def plot_barcodes(barcodes):
    """
    """
    colors = [mcd.XKCD_COLORS['xkcd:'+c]
          for c in ['emerald green', 'tealish', 'peacock blue', 
                    'grey', 'brown', 'red', 'yellow']]
    plt.figure(figsize=(15,6))
    for i, row in barcodes.iterrows():
        sys.stdout.write('\rPlotting barcode {}/{}'.\
                         format(str(i),len(barcodes)))
        sys.stdout.flush()
        birth = row['birth']
        death = row['death']
        x = [birth, 2050] if death==np.inf else\
            [birth, death]
        plt.plot(x, i*np.ones(len(x)), colors[row['dim']])
        if death != np.inf:
            plt.plot(death, i, 'r.')
    plt.axvline(x=2040, linestyle='--', color=mcd.XKCD_COLORS['xkcd:grey'])
    plt.gca().axes.yaxis.set_ticklabels([])
def plot_persistence_diagram(barcodes):
    colors = [mcd.XKCD_COLORS['xkcd:'+c]
      for c in ['emerald green', 'tealish', 'peacock blue', 
                'grey', 'brown', 'red', 'yellow']]
    plt.figure(figsize=(10,10))
    for dim in set(barcodes['dim']):
        data = barcodes.loc[barcodes['dim']==dim]
        data.loc[data['death']==np.inf,'death'] = 2030
        plt.plot(data['birth'], data['death'], '.')
    x = [barcodes['birth'].min(),
         barcodes.loc[barcodes['death']!=np.inf,'death'].max()]
    print(x)
    plt.plot(x, [2030, 2030], '--')
%time plot_barcodes(network.barcodes)
plt.title(topic)
# %time plot_persistence_diagram(barcodes.loc[barcodes['birth']>1900])<jupyter_output><empty_output><jupyter_text>Barcode statistics<jupyter_code>births = {n: [0]*max(network.barcodes['dim']) for n in sorted(network.nodes)}
deaths = {n: [0]*max(network.barcodes['dim']) for n in sorted(network.nodes)}
for row in network.barcodes.iterrows():
    birth_nodes = row[1]['birth nodes']
    death_nodes = row[1]['death nodes']
    for node in birth_nodes:
        births[node][row[1]['dim']-1] += 1
    for node in death_nodes:
        deaths[node][row[1]['dim']-1] += 1
counts = {'names': list(range(len(network.nodes)))*2,
          'type': ['births']*len(network.nodes) + ['deaths']*len(network.nodes),
          'value': [sum(births[n]) for n in network.nodes] +\
                   [sum(deaths[n]) for n in network.nodes]}
plt.figure(figsize=(15,5))
sns.scatterplot(x='names', y='value', hue='type', data=counts)
plt.figure(figsize=(15,3))
sns.distplot([sum(x) for x in births.values()], kde=False, rug=True)
plt.figure(figsize=(15,3))
sns.distplot([sum(x) for x in deaths.values()], kde=False, rug=True)
fig = plt.figure(figsize=(5,5))
hist, edges = np.histogram([sum(x) for x in births.values()], 1000)
ax = sns.scatterplot('x', 'y', data={'x': edges[:-1],
                                     'y': hist}, alpha=0.8)
hist, edges = np.histogram([sum(x) for x in deaths.values()], 1000)
sns.scatterplot('x', 'y', data={'x': edges[:-1],
                                'y': hist}, alpha=0.8)
ax.set_yscale('log')
ax.set_ylim([.5,10000])
ax.set_xscale('log')
ax.set_ylim([.5,10000])
ax.legend(['birth', 'death'])
ax.set_xlabel('number of cavities')
ax.set_ylabel('P(number of cavities)')
[n for n, o in births.items() if sum(o)>400]
[n for n, o in deaths.items() if sum(o)>400]
[n for n in network.graph.nodes if len(network.graph[n])>200]
lifetimes = np.array(network.barcodes['death']-network.barcodes['birth'])
lifetimes[lifetimes >= 1E308] = 0
hist, edges = np.histogram([sum(lifetimes)
                            for lifetimes in deaths.values()], 1000)
ax = sns.scatterplot('x', 'y', data={'x': edges[:-1],
                                     'y': hist})
ax.set_yscale('log')
ax.set_xscale('log')
ax.set_xlim(.5,10000)
ax.set_ylim(.5,10000)
ax.set_aspect('equal', 'box')
ax.set_xlabel('lifetime')
ax.set_ylabel('P(lifetime)')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1920384076815363, 'fraction_numerical': 0.027005401080216044, 'mean_word_length': 3.9751243781094527, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 22, 'https://': 5, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '466578', 'license': 'MIT', 'n_tokens_mistral': 1994, 'n_tokens_neox': 1845, 'n_words': 330, 'path': 'tests/test-persistent-homology.ipynb', 'repo_name': 'erteich/wikinet', 'size': '4999'}