<jupyter_start><jupyter_text>For getting web data.<jupyter_code>from bs4 import BeautifulSoup
import requests<jupyter_output><empty_output><jupyter_text>For network analysis.<jupyter_code>import networkx as nx<jupyter_output><empty_output><jupyter_text>For file handling<jupyter_code>import os, re<jupyter_output><empty_output><jupyter_text>Stats.<jupyter_code>from scipy import stats<jupyter_output><empty_output><jupyter_text>Order data.<jupyter_code>year_order = ['2015','2017','2019','2021']

year_quarter_order = ['{0}Q{1}'.format(year,quarter) for year in range(2015,2022) for quarter in range(1,5)]
year_quarter_order = year_quarter_order[:-1]

qy_pos_map = {v:k for k,v in dict(enumerate(year_quarter_order)).items()}

year_congress_mapping = {'2015':114,'2016':114,'2017':115,'2018':115,'2019':116,'2020':116,'2021':117}<jupyter_output><empty_output><jupyter_text>Member office dataGet the locations of the previous "MemberData" for each Congress from the Internet Archive's Wayback Machine.<jupyter_code>responses_d = {}
responses_d['2015'] = requests.get('http://archive.org/wayback/available?url=https://clerk.house.gov/xml/lists/MemberData.xml&timestamp=20150201').json()
responses_d['2017'] = requests.get('http://archive.org/wayback/available?url=https://clerk.house.gov/xml/lists/MemberData.xml&timestamp=20170201').json()
responses_d['2019'] = requests.get('http://archive.org/wayback/available?url=https://clerk.house.gov/xml/lists/MemberData.xml&timestamp=20190201').json()
responses_d['2021'] = requests.get('http://archive.org/wayback/available?url=https://clerk.house.gov/xml/lists/MemberData.xml&timestamp=20210201').json()<jupyter_output><empty_output><jupyter_text>Get the content of these MemberData fields.<jupyter_code># Container
content_dfs = {}

# String for retrieving content
content_string = 'https://web.archive.org/web/{0}if_/https://clerk.house.gov/xml/lists/MemberData.xml'

# Loop through each year
for _year, _response in responses_d.items():
    
    # Pull timestamp
    _timestamp = _response['archived_snapshots']['closest']['timestamp']
    
    # Retrieve content
    _content = requests.get(content_string.format(_timestamp)).text
    
    # Parse XML into a DataFrame
    _df = pd.read_xml(_content,xpath='./members/*/member-info')
    
    # Extract Congress number
    _df['congress-num'] =  BeautifulSoup(_content).find_all('congress-num')[0].text
    
    content_dfs[_year] = _df
# Clean up index
member_data_df = pd.concat(content_dfs,names=['year']).reset_index(level=0).reset_index(drop=True)

# Concat the office-building and office-room
member_data_df['office-full'] = member_data_df['office-building'] + ' ' + member_data_df['office-room'].astype('str')

member_data_df.head()<jupyter_output><empty_output><jupyter_text>If members are not incumbents or haven't served in previous congresses, they have a 0.<jupyter_code>member_data_df['prior-congress'].value_counts()
member_data_df['incumbent'] = member_data_df['prior-congress'].astype(int) == member_data_df['congress-num'].astype(int) - 1<jupyter_output><empty_output><jupyter_text>Save to disk.<jupyter_code>member_data_df.to_csv('member_data_2015_2021.csv',index=False)<jupyter_output><empty_output><jupyter_text>Make a map of bioguideIDs to official names.<jupyter_code>bioguide_name_map = member_data_df.set_index('bioguideID')['official-name'].to_dict()
bioguide_caucus_map = member_data_df.set_index('bioguideID')['caucus'].to_dict()<jupyter_output><empty_output><jupyter_text>Offices by yearPivot data to get offices for each member by year.<jupyter_code># Pivot to get officials' offices
member_offices_annual = pd.pivot_table(
    data = member_data_df,
    index = 'bioguideID',
    columns = 'year',
    values = 'office-full',
    aggfunc = 'sum'
).reset_index()

# Map official names from bioguideIDs
member_offices_annual['official-name'] = member_offices_annual['bioguideID'].map(bioguide_name_map)

# Inspect
member_offices_annual.head()<jupyter_output><empty_output><jupyter_text>Melt data down.<jupyter_code>member_offices_annual_melt = pd.melt(
    member_offices_annual,
    id_vars = ['bioguideID','official-name'],
    var_name='year',
    value_name='office').sort_values(['bioguideID','year']).reset_index(drop=True)

# Drop missing values
member_offices_annual_melt = member_offices_annual_melt.dropna(subset=['office'])

# Inspect
print(member_offices_annual_melt.shape)
member_offices_annual_melt.head()<jupyter_output>(1755, 4)<jupyter_text>Are members incumbents?<jupyter_code>incumbents = pd.pivot_table(
    data = member_data_df,
    index = ['bioguideID','official-name'],
    columns = ['year'],
    values = 'incumbent',
    aggfunc = 'sum'
).stack().dropna().reset_index()

incumbents.rename(columns={0:'incumbent'},inplace=True)<jupyter_output><empty_output><jupyter_text>Do members have the same office as the previous congress?<jupyter_code>gb_office = member_offices_annual_melt.groupby('bioguideID')['office']
member_offices_annual_melt['office-same'] = gb_office.shift(1) == gb_office.shift(0)<jupyter_output><empty_output><jupyter_text>Join together.<jupyter_code>melted_office_df = pd.merge(
    left = member_offices_annual_melt,
    right = incumbents,
    left_on = ['bioguideID','year','official-name'],
    right_on = ['bioguideID','year','official-name'],
    how = 'left'
)

melted_office_df.head()<jupyter_output><empty_output><jupyter_text>Aggregating and counting unique offices and congresses.<jupyter_code># Define aggregation dictionary
agg_d = {'year':['nunique','min','max'],'office':'nunique'}

# Groupby-aggregation
member_years_offices = member_offices_annual_melt.groupby('bioguideID').agg(agg_d)

# Rename columns
member_years_offices.columns = ['congresses','min_year','max_year','unique_offices']

# Map in official names from bioguideIDs
member_years_offices['official-name'] = member_years_offices.index.map(bioguide_name_map)

# Inspect
member_years_offices.head()
g = sb.catplot(
    data=member_years_offices,
    x='congresses',
    y='unique_offices',
    color='tab:blue',
    kind = 'bar'
)

ax = g.axes.flatten()[0]
ax.set_ylim((0,2.4))
ax.set_yticks(range(0,3))
ax.set_xlabel('Terms')
ax.set_ylabel('Unique offices')
ax.set_title('Office moves for U.S. House members')
plt.tight_layout()
plt.savefig('office_moves_terms.png',facecolor='w',bbox_inches='tight')
g = sb.catplot(
    data = member_years_offices,
    x = 'min_year',
    y = 'unique_offices',
    kind = 'bar',
    order = year_order,
    color = 'tab:green'
)

ax = g.axes.flatten()[0]
ax.set_ylim((0,2.4))
ax.set_yticks(range(0,3))
ax.set_xlabel('First term')
ax.set_ylabel('Unique offices')
ax.set_title('Office moves for by term')
plt.tight_layout()
plt.savefig('office_moves_first_term.png',facecolor='w',bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Edgelist<jupyter_code>transitions_15_17 = member_offices_annual.copy()[['bioguideID','2015','2017']]
transitions_15_17.columns = ['bioguideID','from','to']
transitions_15_17['year'] = '2015->2017'
transitions_17_19 = member_offices_annual.copy()[['bioguideID','2017','2019']]
transitions_17_19.columns = ['bioguideID','from','to']
transitions_15_17['year'] = '2017->2019'
transitions_19_21 = member_offices_annual.copy()[['bioguideID','2019','2021']]
transitions_19_21.columns = ['bioguideID','from','to']
transitions_19_21['year'] = '2019->2021'

transitions_el = pd.concat([transitions_15_17,transitions_17_19,transitions_19_21]).dropna(subset=['from','to'],how='any')
transitions_el.head()
transitions_wel = transitions_el.groupby(['from','to']).agg({'year':'nunique'}).reset_index()

# Filter out 0-weights and self-loops
c0 = transitions_wel['year'] > 0
c1 = transitions_wel['from'] != transitions_wel['to']
transitions_wel = transitions_wel[c0 & c1]
transitions_wel.columns = ['source','target','weight']
transitions_wel.head()
g = nx.from_pandas_edgelist(
    df = transitions_wel,
    edge_attr = True,
    create_using=nx.DiGraph
)
lcc_nodes = max(nx.connected_components(g.to_undirected()), key=len)
lcc_g = nx.subgraph(g,lcc_nodes)
pos = nx.layout.spring_layout(g)
len(lcc_g)/len(g)
f,ax = plt.subplots(figsize=(6,6))

nx.draw_networkx(
    g,
    with_labels = False,
    node_size = 50,
    pos = pos,
    ax = ax
)

f.savefig('hairball.png',facecolor='w')<jupyter_output><empty_output><jupyter_text>Disbursement dataUse the ProPublica "[House Office Expenditure Data](https://projects.propublica.org/represent/expenditures)" databases.<jupyter_code>files = sorted([f for f in os.listdir('./disbursement_data/') if '.csv' in f])
disbursement_dfs = {}

for f in files:
    _quarter = f[:6]
    _df = pd.read_csv(
        './disbursement_data/' + f,
        encoding='latin1',
        low_memory=False,
        thousands=','
    )    
    
    _df.columns = [col.strip() for col in _df.columns]
        
    # Rename column
    if 'SORT SUBTOTAL DESCRIPTION' in _df.columns:
        _df = _df.rename(columns={'SORT SUBTOTAL DESCRIPTION':'CATEGORY'})
        
    disbursement_dfs[_quarter] = _df
# Concatenate CSV files
all_db_df = pd.concat(disbursement_dfs,names=['YEAR-QUARTER']).reset_index(level=0)

# Extract and over-write inconsistent QUARTER and YEAR values
all_db_df['QUARTER'] = all_db_df['YEAR-QUARTER'].str.slice(4)
all_db_df['YEAR'] = all_db_df['YEAR-QUARTER'].str.slice(0,4)

# Different category values
all_db_df.replace(
    {'CATEGORY':
     {'RENT, COMMUNICATION, UTILITIES':'RENT COMMUNICATION UTILITIES',
      'RENT  COMMUNICATION  UTILITIES':'RENT COMMUNICATION UTILITIES'}
    },
    inplace=True
)

# Write to disk
all_db_df.to_csv('all_disbursements_2015_2021.csv',index=False)

# Inspect
print(all_db_df.shape)
all_db_df.head()<jupyter_output>(2774751, 22)<jupyter_text>Exploring column values to help with cleanup and alignment<jupyter_code>col = 'CATEGORY'
all_db_df[col].value_counts(), "{:,}".format(all_db_df.loc[all_db_df[col].notnull(),col].shape[0])
col = 'PURPOSE'
all_db_df[col].value_counts(), "{:,}".format(all_db_df.loc[all_db_df[col].notnull(),col].shape[0])
col = 'TRANSCODELONG'
all_db_df[col].value_counts(), "{:,}".format(all_db_df.loc[all_db_df[col].notnull(),col].shape[0])
col = 'PROGRAM'
all_db_df[col].value_counts(), "{:,}".format(all_db_df.loc[all_db_df[col].notnull(),col].shape[0])
col = 'SORT SEQUENCE'
all_db_df[col].value_counts(), "{:,}".format(all_db_df.loc[all_db_df[col].notnull(),col].shape[0])
col = 'TRANSACTION DATE'
all_db_df[col].value_counts(), "{:,}".format(all_db_df.loc[all_db_df[col].notnull(),col].shape[0])<jupyter_output><empty_output><jupyter_text>Filtering dataFiltering data down to bioguideIDs.<jupyter_code>c0 = all_db_df['BIOGUIDE_ID'].notnull()
c1 = ~all_db_df['SORT SEQUENCE'].astype(str).str.contains('TOTAL')
bioguide_df = all_db_df.copy()[c0 & c1]
bioguide_df['AMOUNT'].fillna(0,inplace=True)

# Extract year and quarter as separate values
bioguide_df['YEAR'] = bioguide_df['YEAR-QUARTER'].str.slice(0,4)
bioguide_df['QUARTER'] = bioguide_df['YEAR-QUARTER'].str.slice(4)

print(bioguide_df.shape)
bioguide_df.head()<jupyter_output>(2100413, 22)<jupyter_text>Pivot to members on index, quarters on columns, and total spending.<jupyter_code>quarterly_spending_df = pd.pivot_table(
    data = bioguide_df,
    index = ['BIOGUIDE_ID','YEAR-QUARTER'],
    columns = 'CATEGORY',
    values = 'AMOUNT',
    aggfunc = 'sum',
)

quarterly_spending_df.head(20)
# Reshape
quarterly_spending_melt = quarterly_spending_df.stack().reset_index()

# Rename columns
quarterly_spending_melt.columns = ['BIOGUIDE_ID','YEAR-QUARTER','CATEGORY','AMOUNT']

# Add names of politicians
quarterly_spending_melt['NAME'] = quarterly_spending_melt['BIOGUIDE_ID'].map(bioguide_name_map)

# Sort the DataFrame by bioguide name, year-quarter, and category
quarterly_spending_melt.sort_values(['BIOGUIDE_ID','YEAR-QUARTER','CATEGORY'],inplace=True)

# Extract year and quarter as separate values
quarterly_spending_melt['YEAR'] = quarterly_spending_melt['YEAR-QUARTER'].str.slice(0,4)
quarterly_spending_melt['QUARTER'] = quarterly_spending_melt['YEAR-QUARTER'].str.slice(4)

# Inspect
quarterly_spending_melt.head()
# Filter out rare categories
_df = quarterly_spending_melt[~quarterly_spending_melt['CATEGORY'].isin(['BENEFITS TO FORMER PERSONNEL','PERSONNEL BENEFITS'])]

# Make a catplot
g = sb.catplot(
    data = _df,
    x = 'AMOUNT',
    y = 'CATEGORY',
    hue = 'QUARTER',
    kind = 'bar',
    hue_order = ['Q1','Q2','Q3','Q4'],
    aspect = 1.5,
    palette = 'tab10',
    orient = 'h'
)

# Flatten and extract axes object
ax = g.axes.flatten()[0]

# Update axes properties
ax.set_xscale('log')
ax.set_xlim((1e1,1e6))
ax.set_ylabel('')

plt.savefig('total_disbursements_by_category_and_quarter.png',facecolor='w')
# Filter out rare categories
_df = quarterly_spending_melt[~quarterly_spending_melt['CATEGORY'].isin(['BENEFITS TO FORMER PERSONNEL','PERSONNEL BENEFITS'])]

# Make a catplot
g = sb.catplot(
    data = _df,
    x = 'AMOUNT',
    y = 'CATEGORY',
    hue = 'YEAR',
    kind = 'bar',
    hue_order = ['2015','2016','2017','2018','2019','2020','2021'],
    aspect = 1.5,
    palette = 'tab10',
    orient = 'h'
)

# Flatten and extract axes object
ax = g.axes.flatten()[0]

# Update axes properties
ax.set_xscale('log')
ax.set_xlim((1e1,1e6))
ax.set_ylabel('')

plt.savefig('total_disbursements_by_category_and_year.png',facecolor='w')
g = sb.catplot(
    data = quarterly_spending_melt,
    x = 'YEAR-QUARTER',
    y = 'AMOUNT',
    hue = 'CATEGORY',
    kind = 'point',
    order = year_quarter_order,
    aspect = 2,
    palette = 'tab10',
    kwargs = {'set_yscale':'log'}
)

ax = g.axes.flatten()[0]
ax.set_yscale('log')
ax.set_xlabel('')
ax.set_ylabel('Total (dollars)')
ax.set_title('Quarterly disbursements by category')
ax.set_xticklabels(ax.get_xticklabels(), rotation = 90)

for _new in ['2015Q1','2017Q1','2019Q1','2021Q1']:
    ax.axvline(qy_pos_map[_new],c='k',ls='--')


plt.savefig('quarterly_disbursements_by_category.png',facecolor='w',bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Congress spending<jupyter_code>bioguide_df.head()
annual_spending_df = pd.pivot_table(
    data = bioguide_df,
    index = ['BIOGUIDE_ID','YEAR'],
    columns = 'CATEGORY',
    values = 'AMOUNT',
    aggfunc = 'sum',
)

annual_spending_df.reset_index(inplace=True)
annual_spending_df['congress'] = annual_spending_df['YEAR'].map(year_congress_mapping)

annual_spending_df['TOTAL'] = annual_spending_df[annual_spending_df.columns[2:-1]].sum(1)
merged_df = pd.merge(
    left = annual_spending_df,
    right = melted_office_df,
    left_on = ['BIOGUIDE_ID','YEAR'],
    right_on = ['bioguideID','year'],
    how = 'left'
)

merged_df.fillna(method='ffill',inplace=True)

merged_df['caucus'] = merged_df['BIOGUIDE_ID'].map(bioguide_caucus_map)

merged_df.head()<jupyter_output><empty_output><jupyter_text>Democrats spend more on<jupyter_code>c_dem = merged_df['caucus'] == 'D'
c_rep = merged_df['caucus'] == 'R'
c_incumb = merged_df['incumbent'] == 1
c_newbie = merged_df['incumbent'] == 0
category = 'TOTAL'
print(stats.ttest_ind(merged_df.loc[c_dem,category],merged_df.loc[c_rep,category]))
print(stats.ttest_ind(merged_df.loc[c_incumb,category],merged_df.loc[c_newbie,category]))

g = sb.catplot(
    data = merged_df,
    x = 'incumbent',
    y = category,
    hue = 'caucus',
    kind = 'bar',
    palette = sb.set_palette(['red','blue'])
)

ax = g.axes.flatten()[0]
ax.set_title('Disbursements for {0},\nby party and incumbency'.format(category.title()))
ax.set_ylabel('Disbursements (dollars)')
ax.set_xticklabels(['Non-incumbents','Incumbents'])
ax.set_xlabel('')
g._legend.set_title('Party')

plt.savefig('spending_{0}.png'.format('_'.join(category.lower().split(' '))),facecolor='w',bbox_inches='tight')
category = 'PERSONNEL COMPENSATION'
print(stats.ttest_ind(merged_df.loc[c_dem,category],merged_df.loc[c_rep,category]))
print(stats.ttest_ind(merged_df.loc[c_incumb,category],merged_df.loc[c_newbie,category]))

g = sb.catplot(
    data = merged_df,
    x = 'incumbent',
    y = category,
    hue = 'caucus',
    kind = 'bar',
    palette = sb.set_palette(['red','blue'])
)

ax = g.axes.flatten()[0]
ax.set_title('Disbursements for {0},\nby party and incumbency'.format(category.title()))
ax.set_ylabel('Disbursements (dollars)')
ax.set_xticklabels(['Non-incumbents','Incumbents'])
ax.set_xlabel('')
g._legend.set_title('Party')

plt.savefig('spending_{0}.png'.format('_'.join(category.lower().split(' '))),facecolor='w',bbox_inches='tight')
category = 'SUPPLIES AND MATERIALS'
print(stats.ttest_ind(merged_df.loc[c_dem,category],merged_df.loc[c_rep,category]))
print(stats.ttest_ind(merged_df.loc[c_incumb,category],merged_df.loc[c_newbie,category]))

g = sb.catplot(
    data = merged_df,
    x = 'incumbent',
    y = category,
    hue = 'caucus',
    kind = 'bar',
    palette = sb.set_palette(['red','blue'])
)

ax = g.axes.flatten()[0]
ax.set_title('Disbursements for {0},\nby party and incumbency'.format(category.title()))
ax.set_ylabel('Disbursements (dollars)')
ax.set_xticklabels(['Non-incumbents','Incumbents'])
ax.set_xlabel('')
g._legend.set_title('Party')

plt.savefig('spending_{0}.png'.format('_'.join(category.lower().split(' '))),facecolor='w',bbox_inches='tight')
category = 'RENT COMMUNICATION UTILITIES'
print(stats.ttest_ind(merged_df.loc[c_dem,category],merged_df.loc[c_rep,category]))
print(stats.ttest_ind(merged_df.loc[c_incumb,category],merged_df.loc[c_newbie,category]))

g = sb.catplot(
    data = merged_df,
    x = 'incumbent',
    y = category,
    hue = 'caucus',
    kind = 'bar',
    palette = sb.set_palette(['red','blue'])
)

ax = g.axes.flatten()[0]
ax.set_title('Disbursements for {0},\nby party and incumbency'.format(category.title()))
ax.set_ylabel('Disbursements (dollars)')
ax.set_xticklabels(['Non-incumbents','Incumbents'])
ax.set_xlabel('')
g._legend.set_title('Party')

plt.savefig('spending_{0}.png'.format('_'.join(category.lower().split(' '))),facecolor='w',bbox_inches='tight')<jupyter_output>Ttest_indResult(statistic=17.40693001805839, pvalue=4.767388863920894e-65)
Ttest_indResult(statistic=4.136426456719944, pvalue=3.6087025839327784e-05)<jupyter_text>Republicans spend more on<jupyter_code>category = 'TRAVEL'
print(stats.ttest_ind(merged_df.loc[c_dem,category],merged_df.loc[c_rep,category]))
print(stats.ttest_ind(merged_df.loc[c_incumb,category],merged_df.loc[c_newbie,category]))

g = sb.catplot(
    data = merged_df,
    x = 'incumbent',
    y = category,
    hue = 'caucus',
    kind = 'bar',
    palette = sb.set_palette(['red','blue'])
)

ax = g.axes.flatten()[0]
ax.set_title('Disbursements for {0},\nby party and incumbency'.format(category.title()))
ax.set_ylabel('Disbursements (dollars)')
ax.set_xticklabels(['Non-incumbents','Incumbents'])
ax.set_xlabel('')
g._legend.set_title('Party')

plt.savefig('spending_{0}.png'.format('_'.join(category.lower().split(' '))),facecolor='w',bbox_inches='tight')
category = 'FRANKED MAIL'
print(stats.ttest_ind(merged_df.loc[c_dem,category],merged_df.loc[c_rep,category]))
print(stats.ttest_ind(merged_df.loc[c_incumb,category],merged_df.loc[c_newbie,category]))

g = sb.catplot(
    data = merged_df,
    x = 'incumbent',
    y = category,
    hue = 'caucus',
    kind = 'bar',
    palette = sb.set_palette(['red','blue'])
)

ax = g.axes.flatten()[0]
ax.set_title('Disbursements for {0},\nby party and incumbency'.format(category.title()))
ax.set_ylabel('Disbursements (dollars)')
ax.set_xticklabels(['Non-incumbents','Incumbents'])
ax.set_xlabel('')
g._legend.set_title('Party')

plt.savefig('spending_{0}.png'.format('_'.join(category.lower().split(' '))),facecolor='w',bbox_inches='tight')
category = 'PRINTING AND REPRODUCTION'
print(stats.ttest_ind(merged_df.loc[c_dem,category],merged_df.loc[c_rep,category]))
print(stats.ttest_ind(merged_df.loc[c_incumb,category],merged_df.loc[c_newbie,category]))

g = sb.catplot(
    data = merged_df,
    x = 'incumbent',
    y = category,
    hue = 'caucus',
    kind = 'bar',
    palette = sb.set_palette(['red','blue'])
)

ax = g.axes.flatten()[0]
ax.set_title('Disbursements for {0},\nby party and incumbency'.format(category.title()))
ax.set_ylabel('Disbursements (dollars)')
ax.set_xticklabels(['Non-incumbents','Incumbents'])
ax.set_xlabel('')
g._legend.set_title('Party')

plt.savefig('spending_{0}.png'.format('_'.join(category.lower().split(' '))),facecolor='w',bbox_inches='tight')<jupyter_output>Ttest_indResult(statistic=-3.6976558556987036, pvalue=0.0002210721984712456)
Ttest_indResult(statistic=-15.384590170210526, pvalue=9.133834928793653e-52)<jupyter_text>Explaining why<jupyter_code>merged_df.loc[c_newbie & c_rep,'BIOGUIDE_ID'].unique()
bioguide_df.head(2)
newbie_rep = merged_df.loc[c_newbie & c_dem,'BIOGUIDE_ID'].unique()
c0 = bioguide_df['BIOGUIDE_ID'].isin(newbie_rep)
c1 = bioguide_df['CATEGORY'] == 'PRINTING AND REPRODUCTION'
bioguide_df.loc[(c0 & c1),['BIOGUIDE_ID','CATEGORY','PAYEE','PURPOSE','AMOUNT','RECIP (orig.)']].sort_values('AMOUNT',ascending=False).head(20)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16642884322678844, 'fraction_numerical': 0.02696917808219178, 'mean_word_length': 6.798590504451038, 'pattern_counts': {'":': 0, '<': 96, '<?xml version=': 0, '>': 100, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '237304', 'license': 'MIT', 'n_tokens_mistral': 8719, 'n_tokens_neox': 8168, 'n_words': 1054, 'path': 'Module Assignments/MA02/MA02 - Office spending.ipynb', 'repo_name': 'cuinfoscience/INFO3402-Spring2022', 'size': '21024'}