<jupyter_start><jupyter_text>Extra features<jupyter_code># Add BipartitePandas to system path, do not run this
# import sys
# sys.path.append('../../..')
# Import the BipartitePandas package 
# (Make sure you have installed it using pip install bipartitepandas)
import bipartitepandas as bpd
# For the example, we simulate bipartite labor data
sim_data = bpd.SimBipartite().sim_network()[['i', 'j', 'y', 't']]
bdf = bpd.BipartiteLong(sim_data)
bdf = bdf.clean_data()
display(bdf)<jupyter_output><empty_output><jupyter_text>Summary statistics<jupyter_code># We can get some summary statistics by running .summary()
sim_data = bpd.SimBipartite().sim_network()[['i', 'j', 'y', 't']]
bdf = bpd.BipartiteLong(sim_data)
bdf.summary()
bdf = bdf.clean_data()
bdf.summary()
# We can easily retrieve the number of workers, firms, and clusters (which is None because the data is not clustered)
print(bdf.n_workers())
print(bdf.n_firms())
print(bdf.n_clusters())<jupyter_output>10000
197
None<jupyter_text>Connected set options<jupyter_code># Can compute the largest connected set of firms
bdf = bpd.BipartiteLong(sim_data)
bdf = bdf[bdf['i'] < 200]
bdf = bdf.clean_data({'connectedness': 'connected'})
display(bdf)
# Can compute the largest biconnected set of firms
bdf = bpd.BipartiteLong(sim_data)
bdf = bdf[bdf['i'] < 200]
bdf = bdf.clean_data({'connectedness': 'biconnected'})
display(bdf)
# Can skip connectedness
bdf = bpd.BipartiteLong(sim_data)
bdf = bdf[bdf['i'] < 200]
bdf = bdf.clean_data({'connectedness': None})
display(bdf)<jupyter_output><empty_output><jupyter_text>Original worker and firm ids<jupyter_code># If you set include_id_reference_dict=True, BipartitePandas saves dataframes linking original column ids to contiguous column ids
try:
    display(bdf.id_reference_dict['i'])
except:
    print('Notice this fails, as include_id_reference_dict has not been set to True')
bdf = bpd.BipartiteLong(sim_data, include_id_reference_dict=True)
bdf = bdf.clean_data()
display(bdf.id_reference_dict['i']) # Notice it's empty as the ids were originally contiguous
bdf = bdf[bdf['i'] < 200]
bdf = bdf.clean_data()
display(bdf.id_reference_dict['i'])
bdf = bdf[bdf['i'] > 99]
bdf = bdf.clean_data()
display(bdf.id_reference_dict['i']) # Now ids are approximately - 100 as we dropped the first 100 rows (but because some workers might no longer in the largest connected set of firms, more than 100 rows can be dropped)
display(bdf.id_reference_dict['j']) # Notice firm ids adjusted from both steps
# Finally, we can return a Pandas DataFrame with the original columns merged in
display(bdf.original_ids())<jupyter_output>Notice this fails, as include_id_reference_dict has not been set to True<jupyter_text>Filling in missing years as unemployed<jupyter_code># BipartitePandas can fill in missing years as unemployed (j=-1) for long data
sim_data = bpd.SimBipartite().sim_network()[['i', 'j', 'y', 't']]
bdf = bpd.BipartiteLong(sim_data)
bdf = bdf[(bdf['t'] == 1) | (bdf['t'] == 5)] # Drop periods 2-4
bdf = bdf.clean_data()
display(bdf.fill_periods()) # Can specify fill_j=x instead of -1 and fill_y=y instead of NaN<jupyter_output><empty_output><jupyter_text>Uncollapsing data (converting from collapsed long to long)<jupyter_code># BipartitePandas allows you to un-collapse collapsed long data (variables assumed constant over spells)
sim_data = bpd.SimBipartite().sim_network()[['i', 'j', 'y', 't']]
bdf = bpd.BipartiteLong(sim_data)
bdf = bdf.clean_data().get_collapsed_long()
display(bdf)
bdf = bdf.uncollapse()
display(bdf)<jupyter_output><empty_output><jupyter_text>Handling i-t (worker-year) duplicates (note perfect duplicates are always dropped)<jupyter_code># Keep max paying job
bdf = bpd.BipartiteLong(sim_data)
append_val = bdf.iloc[0].copy()
append_val['y'] = 2 * append_val['y']
bdf = bdf.append(append_val)
bdf[['i', 'j', 't']] = bdf[['i', 'j', 't']].astype(int)
bdf = bdf.clean_data({'i_t_how': 'max'})
display(bdf)
# Keep max sum over i-j-t (worker-firm-year) duplicates
bdf = bpd.BipartiteLong(sim_data)
append_val = bdf.iloc[0].copy()
append_val['y'] = 2 * append_val['y']
bdf = bdf.append(append_val)
bdf[['i', 'j', 't']] = bdf[['i', 'j', 't']].astype(int)
bdf = bdf.clean_data({'i_t_how': 'sum'})
display(bdf)
# Keep max mean over i-j-t (worker-firm-year) duplicates
bdf = bpd.BipartiteLong(sim_data)
append_val = bdf.iloc[0].copy()
append_val['y'] = 2 * append_val['y']
bdf = bdf.append(append_val)
bdf[['i', 'j', 't']] = bdf[['i', 'j', 't']].astype(int)
bdf = bdf.clean_data({'i_t_how': 'mean'})
display(bdf)<jupyter_output><empty_output><jupyter_text>Column inference<jupyter_code># BipartitePandas does some nice column inference
# Suppose some of our columns are mislabeled - create a col_dict to clarify which columns to correct
sim_data = bpd.SimBipartite().sim_network()[['i', 'j', 'y', 't']]
sim_data = sim_data.rename({'i': 'i1'}, axis=1)
bdf = bpd.BipartiteLong(sim_data, col_dict={'i': 'i1'})
bdf = bdf.clean_data()
display(bdf)
# Suppose we have already clustered - the class determines this automatically
sim_data = bpd.SimBipartite().sim_network()[['i', 'j', 'y', 't']]
sim_data['g'] = 1
bdf = bpd.BipartiteLong(sim_data)
bdf = bdf.clean_data()
display(bdf)
# Now suppose we mislabeled our clusters - the class drops this column
sim_data = bpd.SimBipartite().sim_network()[['i', 'j', 'y', 't']]
sim_data['g1'] = 1
bdf = bpd.BipartiteLong(sim_data)
bdf = bdf.clean_data()
display(bdf)
# We can make sure our columns are included by specifying the corrected labels in col_dict (note that if the variable of interest requires 2 columns, e.g. weight 1 and weight 2 for event study data, BOTH columns are required - if only one column is included, it is automatically dropped.)
sim_data = bpd.SimBipartite().sim_network()[['i', 'j', 'y', 't']]
sim_data['g1'] = 1
bdf = bpd.BipartiteLong(sim_data, col_dict={'g': 'g1'})
bdf = bdf.clean_data()
display(bdf)
# We may also specify a col_dict that overwrites a column label that would normally be inferred, and inference is skipped
sim_data = bpd.SimBipartite().sim_network()[['i', 'j', 'y', 't']]
sim_data = sim_data.rename({'i': 'g'}, axis=1)
bdf = bpd.BipartiteLong(sim_data, col_dict={'i': 'g'})
# Note that even though we have a column named g, because we specify it is the 'i' column it is not inferred as the cluster values
bdf = bdf.clean_data()
display(bdf)
# Dropping an optional column
bdf.gen_m()
display(bdf)
bdf.drop('m')
# Dropping a required column is prevented
bdf.drop('i')<jupyter_output>../../../bipartitepandas/bipartitebase.py:305: UserWarning: i is either (a) a required column and cannot be dropped or (b) a subcolumn that can be dropped, but only by specifying the general column name (e.g. use 'g' instead of 'g1' or 'g2')
  warnings.warn("{} is either (a) a required column and cannot be dropped or (b) a subcolumn that can be dropped, but only by specifying the general column name (e.g. use 'g' instead of 'g1' or 'g2')".format(col))
{'fraction_non_alphanumeric': 0.14425709756449057, 'fraction_numerical': 0.009079118028534372, 'mean_word_length': 6.103377686796315, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '535419', 'license': 'MIT', 'n_tokens_mistral': 2614, 'n_tokens_neox': 2520, 'n_words': 728, 'path': 'docs/source/notebooks/extra_features_examples.ipynb', 'repo_name': 'adamoppenheimer/bipartitepandas', 'size': '6939'}