<jupyter_start><jupyter_text>Data Description<jupyter_code>uscv_edgelist_fpath = os.path.join(DATA_DIR, 'bill_0-1800_edgelist.tsv')
uscv_edgeinfo_fpath = os.path.join(DATA_DIR, 'bill_edgeinfo.tsv')
uscv_nodeinfo_fpath = os.path.join(DATA_DIR, 'bill_nodeinfo.tsv')
edge_df = pd.read_csv(uscv_edgelist_fpath, sep='\t')
edge_df
edgeinfo_df = pd.read_csv(uscv_edgeinfo_fpath, sep='\t')
edgeinfo_df
print(json.loads(edgeinfo_df.iloc[0]['Info']))
nodeinfo_df = pd.read_csv(uscv_nodeinfo_fpath, sep='\t')
nodeinfo_df
print(json.loads(nodeinfo_df.iloc[0]['Info']))<jupyter_output>{'person': 300001, 'state': 'HI', 'district': 2, 'name': 'Rep. <NAME> [D, 1977-1990]', 'party': 'Democrat', 'url': 'https://www.govtrack.us/congress/members/daniel_akaka/300001'}<jupyter_text>Original Vote DataAs we said in the paper, the dataset is constructed from vote data.We give the origin dataset as follows.<jupyter_code>vote_data_path = os.path.join(DATA_DIR, 'bill_a_c_clean.tsv')
vote_df = pd.read_csv(vote_data_path, sep='\t')
vote_df<jupyter_output><empty_output><jupyter_text>Data Analysis  House or Senate<jupyter_code>vote_df['category'] = vote_df['bill_name'].apply(lambda x:x.rpartition('-')[-1][0])
vote_df.category.value_counts()<jupyter_output><empty_output><jupyter_text>Bill  Counts<jupyter_code>t = list(map(lambda x:x.rpartition('-')[-1][0], vote_df['bill_name'].unique().tolist()))
from collections import Counter
c_t = Counter(t)
plt.bar(c_t.keys(), c_t.values())<jupyter_output><empty_output><jupyter_text>Sign Ratio<jupyter_code>vote_df[vote_df['category'] == 'h']['sign'].value_counts()
vote_df[vote_df['category'] == 's']['sign'].value_counts()
vote_df['year'] = vote_df['date'].apply(lambda x: int(x.split('-')[0]))
df_s = vote_df[vote_df['category'] == 's']
df_h = vote_df[vote_df['category'] == 'h']<jupyter_output><empty_output><jupyter_text>Time Span  Senate<jupyter_code>df_s_tmp = df_s.groupby('bill_name').agg({'year': 'mean'}).reset_index()
ax = df_s_tmp['year'].apply(lambda x: x // 2 * 2).value_counts().sort_index().plot(kind='bar', figsize=(30,5), rot=60)
# t = ax.get_xaxis().get_ticklabels()
# t2 = [int(i.get_text()) * 2 for i in t]
# ax.set_xticklabels(t2, rotation=45)<jupyter_output><empty_output><jupyter_text>House<jupyter_code>df_h_tmp = df_h.groupby('bill_name').agg({'year': 'mean'}).reset_index()
ax = df_h_tmp['year'].apply(lambda x: x // 2 * 2).value_counts().sort_index().plot(kind='bar', figsize=(30,5), rot=60)
# t = ax.get_xaxis().get_ticklabels()
# t2 = [int(i.get_text()) * 2 for i in t]
# ax.set_xticklabels(t2, rotation=45)<jupyter_output><empty_output><jupyter_text>Negative Link Ratio with Time  Senate<jupyter_code>df_s1 = df_s[df_s['sign'] == "YEA"].groupby('bill_name').agg({'year': 'mean', 'sign': 'count'}).reset_index()
df_s1.rename(columns={'sign': 'Y_c'}, inplace=True)
df_s2 = df_s[df_s['sign'] == 'NO'].groupby('bill_name').agg({'year': 'mean', 'sign': 'count'}).reset_index()
df_s2.rename(columns={'sign': 'N_c'}, inplace=True)
df_s3 = df_s1.merge(df_s2, on='bill_name')
df_s3['y_ratio'] = df_s3['Y_c']/ (df_s3['N_c'] + df_s3['Y_c'])
df_s3['year_t'] = df_s3['year_x'].apply(lambda x: x//2 * 2)
ax = df_s3[['year_t', 'y_ratio']].groupby('year_t').agg({'y_ratio':'mean'}).plot.bar(figsize=(20,5), rot=60)
# t = ax.get_xaxis().get_ticklabels()<jupyter_output><empty_output><jupyter_text>House<jupyter_code>df_h1 = df_h[df_h['sign'] == "YEA"].groupby('bill_name').agg({'year': 'mean', 'sign': 'count'}).reset_index()
df_h1.rename(columns={'sign': 'Y_c'}, inplace=True)
df_h2 = df_h[df_h['sign'] == 'NO'].groupby('bill_name').agg({'year': 'mean', 'sign': 'count'}).reset_index()
df_h2.rename(columns={'sign': 'N_c'}, inplace=True)
df_h3 = df_h1.merge(df_h2, on='bill_name')
df_h3['y_ratio'] = df_h3['Y_c']/ (df_h3['N_c'] + df_h3['Y_c'])
df_h3['year_t'] = df_h3['year_x'].apply(lambda x: x//2 * 2)
ax = df_h3[['year_t', 'y_ratio']].groupby('year_t').agg({'y_ratio':'mean'}).plot.bar(figsize=(20,5),  rot=60)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22024108488196886, 'fraction_numerical': 0.024861878453038673, 'mean_word_length': 9.032745591939547, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '174086', 'license': 'MIT', 'n_tokens_mistral': 1777, 'n_tokens_neox': 1721, 'n_words': 214, 'path': 'ipynbs/uscv_analysis.ipynb', 'repo_name': 'huangjunjie95/signlens', 'size': '3988'}