<jupyter_start><jupyter_text>Number of papers per author<jupyter_code>filename = "/Users/jsennett/Code/top-k-insights/data/all-author-paper-counts-by-year.csv"
df = pd.read_csv(filename, encoding='mac_roman')
print(len(data), "rows (4.4 million)")
print(df.columns)
df.head()

df.describe()
# Scatter plots of some relevant measures
df.plot.scatter('year', 'num_papers')
# Plot num_papers versus rank
df['rank'] = df['num_papers'].rank(method='first', ascending=False)
df.plot.scatter('rank', 'num_papers')
df['log_rank'] = scipy.log2(df['rank'])
df['log_num_papers'] = scipy.log2(df['num_papers'])
df.plot.scatter('log_rank', 'log_num_papers')<jupyter_output>/Users/jsennett/.pyenv/versions/3.6.1/envs/topk/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: scipy.log2 is deprecated and will be removed in SciPy 2.0.0, use numpy.lib.scimath.log2 instead
  """Entry point for launching an IPython kernel.
/Users/jsennett/.pyenv/versions/3.6.1/envs/topk/lib/python3.6/site-packages/ipykernel_launcher.py:2: DeprecationWarning: scipy.log2 is deprecated and will be removed in SciPy 2.0.0, use numpy.lib.scimath.log2 instead<jupyter_text>All papers<jupyter_code>papers_filename = "/Users/jsennett/Code/top-k-insights/data/all-papers.csv"
p_df = pd.read_csv(papers_filename, encoding='mac_roman', dtype = {'school': str})
print(len(p_df), "rows (4.4 million)")
print(p_df.columns)
p_df.head()
# Insight: positive trend of paper counts over year considering the full set of data
p_df.groupby('year')\
    .agg('count')\
    .reset_index()\
    .rename(columns={'paperid':'number of papers'})\
    .plot.scatter('year', 'number of papers')
# Insight: positive trend of paper counts over year considering the full set of data
p_df[p_df['venue_type'] == 1].groupby('year')\
    .agg('count')\
    .reset_index()\
    .rename(columns={'paperid':'number of papers'})\
    .plot.scatter('year', 'number of papers')
p_df[p_df['venue_type'] == 0]['year'].value_counts()
p_df['M'] = 1
# Insight: positive trend of paper counts over year considering the full set of data
p_df[p_df['venue_type'] == 1].groupby('year')\
    .agg('count')\
    .reset_index()\
    .rename(columns={'paperid':'number of papers'})\
    .plot.scatter('year', 'number of papers')
def scatter(df, dimension):
    return df.groupby(dimension)\
        .agg('count')\
        .reset_index()\
        .rename(columns={'paperid':'number of papers'})\
        .plot.scatter(dimension, 'number of papers')
def rank_scatter(df, dimension):
    rank_df = df.groupby(dimension).agg('count').reset_index()
    rank_df['rank'] = rank_df['M'].rank(method='first', ascending=False)
    return rank_df.plot.scatter(x='rank', y='M', label='rank of ' + dimension)
rank_scatter(p_df, 'year')
venue_ranks['rank'] = venue_ranks['M'].rank(method='first', ascending=False)
venue_ranks.plot.scatter(x='rank', y='M')
p_df['paperid'].rank()
p_df.groupby('venue_name').agg('count').head().rank()

df['rank'] = df['num_papers'].rank(method='first', ascending=False)
result_set = p_df.groupby(['venue_name', 'year']).agg({'M':'sum'}).reset_index()
result_set['M'] = result_set.sort_values("year")['M'].diff()
result_set.head()
result_set['absM'] = abs(result_set['M'])
rank_scatter(result_set, 'absM')
import scipy
result_set['logabsM'] = scipy.log2(result_set['absM'])
result_set['logrank'] = scipy.log2(result_set['M'].rank(method='first', ascending=False))
result_set['rank'] = result_set['M'].rank(method='first', ascending=False)
result_set.plot.scatter('rank', 'M')
result_set[result_set['M'] > 0].plot.scatter('logrank', 'logabsM')
result_set[result_set['M'] > 50].plot.scatter('rank', 'M')
cutoff = 0.2
result_set[result_set['rank'] < cutoff * len(result_set)].plot.scatter('logrank', 'logabsM')
# pct of total:
pct_of_total = p_df.copy()
pct_of_total = pct_of_total.groupby(['venue_name']).agg({'M':'sum'})
pct_of_total = pct_of_total.reset_index(drop=False)
pct_of_total.head()
pct_of_total['M'] = 100 * pct_of_total['M'] / sum(pct_of_total['M'])
pct_of_total['rank'] = pct_of_total['M'].rank(method='first', ascending=False)
pct_of_total.plot.scatter('rank', 'M')
pct_of_total['logrank'] = scipy.log2(pct_of_total['rank'])
pct_of_total['logM'] = scipy.log2(pct_of_total['M'])
pct_of_total.plot.scatter('logrank', 'logM')
pct_of_total.plot.scatter('logrank', 'M')
pct_of_total['M'] = 100 * result_set['M'] / sum(result_set['M'])
result_set['M'] = 100 * result_set[self.measure] / sum(result_set[self.measure])
p_df.head()
# Validate composite extractor result sets
"""
INFO:root:Valid SG/CE combo: subspace({'year': 2000}), dim(venue_type), ce([['count', 'count'], ['pct', 'year']])
INFO:root:extract_result_set({'year': 2000}, venue_type, [['count', 'count'], ['pct', 'year']]
INFO:root:RESULT SET:
     venue_type  year  count mod2         M
10           0  2000  30846    *  2.363180
36           1  2000  36539    *  2.175786
62           3  2000     43    *  0.633844
"""
print(100 * len(p_df[(p_df['year']==2000) & (p_df['venue_type'] == 0)]) / len(p_df[p_df['venue_type']==0]))
print(100 * len(p_df[(p_df['year']==2000) & (p_df['venue_type'] == 1)]) / len(p_df[p_df['venue_type']==1]))
print(100 * len(p_df[(p_df['year']==2000) & (p_df['venue_type'] == 3)]) / len(p_df[p_df['venue_type']==3]))
"""
INFO:root:extract_result_set({'year': 2000}, venue_type, [['count', 'count'], ['delta_prev', 'year']]
returning result set:

    venue_type  year  count mod2     M
1           0  2000  30846    *  4762
3           1  2000  36539    *  3805
5           3  2000     43    *     3
"""
print(len(p_df[(p_df['year'] == 2000) & (p_df['venue_type'] == 0)]) - len(p_df[(p_df['year'] == 1999) & (p_df['venue_type'] == 0)]))
print(len(p_df[(p_df['year'] == 2000) & (p_df['venue_type'] == 1)]) - len(p_df[(p_df['year'] == 1999) & (p_df['venue_type'] == 1)]))
print(len(p_df[(p_df['year'] == 2000) & (p_df['venue_type'] == 3)]) - len(p_df[(p_df['year'] == 1999) & (p_df['venue_type'] == 3)]))
x = pd.DataFrame([10 * np.random.normal() + 15 for _ in range(100)], columns=['M'])
x['M'].hist()
max(x['M'])
scipy.stats.norm.pdf()
(38 - x['M'].mean()) / x['M'].std()
scipy.stats.norm.ppf(100/)
x.rank(method='first', ascending=False)
x.hist()
x['rank'] = x['M'].rank(ascending=True, method='first')
x['pct'] = x['rank'] / (len(x)+1)
x['Z'] = (x['M'] - x['M'].mean()) / x['M'].std()
x['predZ'] = scipy.stats.norm.pdf(x['pct'])
x['pred'] = x['pct'] * x['M'].std() + x['M'].mean()
x['pred'].hist()
x.head(25)
x.head()
xmean = x['M'].mean()
xstd = x["M"].std()
x['rank'] = x['M'].rank(method='first')
x.plot.scatter('rank', 'M')
(1 - scipy.stats.norm.cdf(x_max_Z)) / (.01)
alpha = 1/100
x_max_p = 1 - scipy.stats.norm.cdf(x_max_Z)
print(x_max_p)
significance = max(1 - x_max_p / alpha, 0.0)
print(significance)
x = pd.DataFrame([10 * np.random.normal() + 15 for _ in range(100)], columns=['M'])
x_max = max(x['M'])
x_mean = x.drop(x['M'].idxmax())['M'].mean()
x_std = x.drop(x['M'].idxmax())['M'].std()
x_max_Z = (x_max - x_mean)/x_std
x_max_p = 1 - scipy.stats.norm.cdf(x_max_Z)

x.hist()
plt.axvline(scipy.stats.norm.ppf(.99)  * x_std + x_mean, color='r')
plt.axvline(x_max, color='g')
print("significance:", max(1 - x_max_p / alpha, 0.0))
x = pd.DataFrame([10 * np.random.normal() + 15 for _ in range(100)], columns=['M'])
x_max = max(x['M'])
x_mean = x.drop(x['M'].idxmax())['M'].mean()
x_std = x.drop(x['M'].idxmax())['M'].std()
x_max_Z = (x_max - x_mean)/x_std
x_max_p = 1 - scipy.stats.norm.cdf(x_max_Z)

x.hist()
plt.axvline(scipy.stats.norm.ppf(.99)  * x_std + x_mean, color='r')
plt.axvline(x_max, color='g')
print("significance:", max(1 - x_max_p / alpha, 0.0))
def simulate(n):
    x = pd.DataFrame([10 * np.random.normal() + 15 for _ in range(n)], columns=['M'])
    x_max = max(x['M'])
    alpha = 1/n    
    
    x_mean = x['M'].mean()
    x_std = x['M'].std()
    x_max_Z = (x_max - x_mean)/x_std
    x_max_p = 1 - scipy.stats.norm.cdf(x_max_Z)

    x.hist(density=True)
    plt.axvline(scipy.stats.norm.ppf(1-1/n) * x_std + x_mean, color='r', label='expected max val')
    plt.axvline(x_max, color='g', label='actual max val')

    # plot normal dist
    x_axis = np.arange(min(x['M']), max(x['M']), .001)
    plt.plot(x_axis, scipy.stats.norm.pdf(x_axis,x_mean,x_std))
    # Mean = 0, SD = 2.

    print("significance:", max(1 - x_max_p / alpha, 0.0))
    print("x_max_p", x_max_p)
    print("alpha (1/n)", alpha)
    
simulate(1000)
simulate(1000)
simulate(1000)
simulate(1000)
simulate(1000)
scipy.stats.norm.ppf(.5, x_mean, x_std)
scipy.stats.norm.pdf(x_mean, x_mean, x_std)
x_std
x_min = -25
x_max = 50
mean = 15 
std = 9
x = np.linspace(x_min, x_max, 100)
y = scipy.stats.norm.pdf(x,mean,std) * 1000
plt.plot(x, y)
import numpy as np
from scipy.stats import norm
import matplotlib.pyplot as plt
# Plot the histogram.
plt.hist(data, bins=25, density=True, alpha=0.6, color='g')

# Plot the PDF.
xmin, xmax = plt.xlim()
x = np.linspace(xmin, xmax, 100)
p = norm.pdf(x, mu, std)
plt.plot(x, p, 'k', linewidth=2)
title = "Fit results: mu = %.2f,  std = %.2f" % (mu, std)
plt.title(title)

plt.show()
p_df[p_df['venue_name'] == 'CoRR']
result_set = p_df
analysis_dimension = 'venue_name'
subspace = {'venue_name':'CoRR'}
result_set[result_set[analysis_dimension] == subspace[analysis_dimension]]
p_df = p_df.fillna('')
school_size = p_df.groupby('school').agg({"M":"sum"})
school_size[school_size['M'] > len(p_df) * .01]
p_df = pd.read_csv(papers_filename, encoding='mac_roman', dtype = {'school': str})
len(p_df)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21585532541268004, 'fraction_numerical': 0.03469666701713805, 'mean_word_length': 5.113110539845758, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '72475', 'license': 'MIT', 'n_tokens_mistral': 4172, 'n_tokens_neox': 3897, 'n_words': 676, 'path': 'report/notebooks/Data Exploration.ipynb', 'repo_name': 'jsennett/top-k-insights', 'size': '9511'}