<jupyter_start><jupyter_text>1 Data pull from arXiv====<jupyter_code>def pull_data (category, date_from, date_until):
    scraper = arxivscraper.Scraper(category=category, date_from=date_from,date_until=date_until)
    output = scraper.scrape()
    df = pd.DataFrame([])
    if (output!=1):
        cols = ('id', 'title', 'categories', 'abstract', 'doi', 'created', 'updated', 'authors')
        df = pd.DataFrame(output,columns=cols)
    return(df)
cats = ['stat', 'cs']
years = list(range(2007, 2020))
for category in cats:
    for year in years:
        print(f"\n=====\nQuerying metadata in arXiv category {category}, year {year}\n=====\n")
        df = pd.DataFrame([])
        for month in list(range(1, 13)):
            month_digits = str(month).zfill(2)
            lastday = monthrange(year, month)[1]
            df0 = pull_data(category, f"{year}-{month_digits}-01", f"{year}-{month_digits}-{lastday}")
            df = pd.concat([df, df0])
        df.to_csv(f"arxiv_metadata_{category}_{year}.csv", index=False)
        numrecords = format(len(df), ',')
        print(f"\nSaved {numrecords} total records for {year}, {category} category\n")

print('done')
sdf = pd.concat([pd.read_csv(ff) for ff in list(filter(lambda x: '_stat_' in x, os.listdir('.')))])
cdf = pd.concat([pd.read_csv(ff) for ff in list(filter(lambda x: '_cs_' in x, os.listdir('.')))])
len(sdf) + len(cdf)<jupyter_output><empty_output><jupyter_text>2 Exploratory analysis of selected stat and cs categories for code availability===<jupyter_code>def findcategory (category, df):
    return df.categories.apply(lambda x: category in x)
def findcategories (categories, df):
    return df.categories.apply(lambda x: bool(sum([category in x for category in categories])))
def findterm (term, df):
    return df.abstract.apply(lambda x: term in x)
def findterms (terms, df):
    return df.abstract.apply(lambda x: bool(sum([term in x for term in terms])))
categories = ['stat.th', 'stat.ap', 'stat.ml']
cscategories = ['cs.cv', 'cs.ai']
findcategories(categories + cscategories, pd.concat([sdf, cdf])).sum()
def categorytermcontent (category, terms, df):
    newdf = df[findcategory(category, df)]
    mark = findterms(terms, newdf)
    count = int(mark.sum())
    total = len(newdf)
    pct = 0
    se = 0
    if total > 0:
        pct = float(count) / float(total)
        se = math.sqrt(pct * (1 - pct) / total)
    return {'category' : category, 'terms' : terms, 'count' : count, \
            'percent' : pct, 'total' : total, \
           'se' : se}
def explore_terms (terms, categories, df):
    for term in terms:
        print(f"\n====\nCounts of abstracts with term '{term}':\n====\n")
        for category in categories:
            res = categorytermcontent(category, [term], df)
            print(f"{res['category']}: {res['count']}")
    return(0)
terms = ['github', 'sourceforge', 'open source', 'code', 'r package', 'python module', 'python package', 'r module']
explore_terms(terms, categories, sdf)
cscategories = ['cs.cv', 'cs.ai']
explore_terms(terms, cscategories, cdf)<jupyter_output>====
Counts of abstracts with term 'github':
====

cs.cv: 1420
cs.ai: 293

====
Counts of abstracts with term 'sourceforge':
====

cs.cv: 3
cs.ai: 3

====
Counts of abstracts with term 'open source':
====

cs.cv: 169
cs.ai: 104

====
Counts of abstracts with term 'code':
====

cs.cv: 5119
cs.ai: 1598

====
Counts of abstracts with term 'r package':
====

cs.cv: 1
cs.ai: 13

====
Counts of abstracts with term 'python module':
====

cs.cv: 4
cs.ai: 0

====
Counts of abstracts with term 'python package':
====

cs.cv: 4
cs.ai: 6

====
Counts of abstracts with term 'r module':
====

cs.cv: 121
cs.ai: 31<jupyter_text>3 Charts of code availability trends over time====<jupyter_code>def create_count_time_series (categories, terms, years, df):
    result = []
    for category in categories:
        for term in terms:
            for year in years:
                yearmark = df.created.apply(lambda x: str(year) in x)
                df1 = df[yearmark]
                res = categorytermcontent(category, [term], df1)
                res['year'] = year
                result.append(res)
    return(pd.DataFrame(result))

def create_count_trends (categories, terms, years, df):
    result = []
    for category in categories:
        for year in years:
            yearmark = df.created.apply(lambda x: str(year) in x)
            df1 = df[yearmark]
            res = categorytermcontent(category, terms, df1)
            res['year'] = year
            result.append(res)
    return(pd.DataFrame(result))
counts = create_count_time_series(categories, terms, years, sdf)
ct = create_count_trends(categories, terms, years, sdf)
cst = create_count_trends(cscategories, terms, years, cdf)
def plot_trends (trends, filename='trends.png'):
    fig, ax = plt.subplots(nrows=2, figsize=(9,6))
    for name, group in trends.groupby(['category']):
        group.plot(x='year', y='percent', ax=ax[0], label=name + ' % articles')
        ax[0].fill_between(group.year, group.percent - group.se, group.percent + group.se, alpha=0.35)

    l = ax[0].get_lines()

    count = 0

    for name, group in trends.groupby(['category']):
        group.plot(x='year', y='total', ax=ax[1], label=name + ' # articles', style='--',\
                  color=list(i.get_color() for i in l)[count])
        count+=1

    #ax[0].yaxis.grid(which="major")
    #ax[1].yaxis.grid(which="major")
    
    ax[0].set_ylabel('arXiv articles using one of terms')
    ax[1].set_ylabel('arXiv number of articles')

    vals = ax[0].get_yticks()
    ax[0].set_yticklabels(['{:,.0%}'.format(x) for x in vals])
    
    ax[1].get_yaxis().set_major_formatter(mpl.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
    ax[0].legend(loc='upper left')
    ax[1].legend(loc='upper left')
    plt.savefig(filename, dpi=500, bbox_inches='tight')
plot_trends(ct, filename='stat_trends2007_to_2019.png')
plot_trends(cst, filename='cs_trends2007_to_2019.png')
ctc = create_count_trends(categories + cscategories, terms, years, pd.concat([sdf, cdf]))
plot_trends(ctc, filename='combined_trends_2007_to_2019.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16383357616966165, 'fraction_numerical': 0.018455560951918408, 'mean_word_length': 3.6035767511177346, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '233775', 'license': 'MIT', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2106, 'n_words': 529, 'path': 'arxiv_appers_analysis.ipynb', 'repo_name': 'nesterko/ds_code_availability_on_arxiv', 'size': '6177'}