<jupyter_start><jupyter_text>Create Sci-Hub Access Log journal summaries and top articles datasets<jupyter_code>import os
import json

import pandas
import numpy<jupyter_output><empty_output><jupyter_text>Read and process input data<jupyter_code># Read catalog of Crossref DOIs
path = os.path.join('data', 'doi.tsv.xz')
doi_df = pandas.read_table(path, parse_dates=['issued'])
# Define issued timeperiod to consider articles for journal metrics
start = pandas.to_datetime('2014-01-01')
stop = pandas.to_datetime('2015-09-01')
doi_df = doi_df.query("@start <= issued < @stop")
doi_df.head(3)
path = os.path.join('data', 'scopus-title-to-doi-map.tsv.xz')
title_map_df = pandas.read_table(path)
title_map_df = title_map_df.query("doi in @doi_df.doi")
title_map_df.head(2)
# Read Sci-Hub logs summary
path = os.path.join('download', 'scihub-logs', 'scihub-logs-summary_2015-09_2016-02.tsv.xz')
logs_df = pandas.read_table(path)
logs_df = title_map_df.merge(logs_df, how='left').fillna(0)
columns = ['downloads', 'visitors', 'countries', 'days', 'months']
logs_df[columns] = logs_df[columns].astype(int)
logs_df.head(3)<jupyter_output><empty_output><jupyter_text>Top 100 papers per journal<jupyter_code>top_paper_df = (
    logs_df
    .sort_values(['visitors', 'doi'], ascending=[False, True])
    .groupby('scopus_id')
    .head(100)
    .sort_values('scopus_id', kind='mergesort')  # mergesort is stable
    .reset_index(drop=True)
)
top_paper_df.head(2)
path = os.path.join('download', 'scihub-logs', 'crossref-metadata.tsv.xz')
crossref_df = (
    pandas.read_table(path)
    .query("doi in @top_paper_df.doi")
    [['doi', 'issued', 'authors', 'title']]
)
top_paper_df = top_paper_df.merge(crossref_df)
top_paper_df.head(2)
top_paper_df.scopus_id.nunique()
path = os.path.join('data', 'scihub-log-journal-top-articles.tsv.xz')
top_paper_df.to_csv(path, sep='\t', index=False, compression='xz')<jupyter_output><empty_output><jupyter_text>Quantiles per Journal<jupyter_code>def get_quantiles(df):
    q = numpy.linspace(0, 1, 101)
    quantile_df = df[columns].quantile(q)
    quantile_df.index.rename('quantile', inplace=True)
    quantile_df = quantile_df.reset_index()
    return quantile_df

quantile_df = logs_df.groupby('scopus_id').apply(get_quantiles).reset_index(level='scopus_id')
quantile_df.head(2)
path = os.path.join('data', 'scihub-log-journal-quantiles.tsv.xz')
quantile_df.to_csv(path, sep='\t', index=False, float_format='%.5g', compression='xz')<jupyter_output><empty_output><jupyter_text>Journal metadata from Scopus<jupyter_code>with open('00.configuration.json') as read_file:
    config = json.load(read_file)

# Read Scopus title names and attributes
url = config['scopus_url'] + 'data/titles.tsv'
journal_df = pandas.read_table(url)

url = config['scopus_url'] + 'data/title-attributes.tsv'
journal_df = (
    journal_df
    .merge(pandas.read_table(url))
    [['scopus_id', 'title_name', 'active', 'open_access', 'main_publisher']]
)
journal_df.head(2)<jupyter_output><empty_output><jupyter_text>Journal mean summaries<jupyter_code>def summarize(df):
    series = df[columns].mean()
    series['n_articles_requested'] = sum(df.downloads > 0)
    series['n_articles'] = len(df)
    return series

mean_df = logs_df.groupby('scopus_id').apply(summarize).reset_index()
mean_df = journal_df.merge(mean_df)
for column in 'n_articles_requested', 'n_articles':
    mean_df[column] = mean_df[column].astype(int)
mean_df.head(2)
mean_df.sort_values('visitors', ascending=False).head()
path = os.path.join('data', 'scihub-log-journal-averages.tsv.xz')
mean_df.to_csv(path, sep='\t', index=False, float_format='%.5g', compression='xz')<jupyter_output><empty_output><jupyter_text>Publisher mean summaries<jupyter_code>publisher_mean_df = logs_df.merge(journal_df).groupby('main_publisher').apply(summarize).reset_index()
for column in 'n_articles_requested', 'n_articles':
    publisher_mean_df[column] = publisher_mean_df[column].astype(int)
publisher_mean_df.head(2)
# Most visited publishers who've published over 1000 articles
publisher_mean_df.query("n_articles > 1000").sort_values('visitors', ascending=False).head(10)
path = os.path.join('data', 'scihub-log-publisher-averages.tsv')
publisher_mean_df.to_csv(path, sep='\t', index=False, float_format='%.5g')<jupyter_output><empty_output><jupyter_text>Top articles per publisher<jupyter_code>top_paper_publisher_df = (
    top_paper_df
    .merge(journal_df[['scopus_id', 'title_name', 'main_publisher']])
    .sort_values(['visitors', 'doi'], ascending=[False, True])
    .groupby('main_publisher')
    .head(100)
    .sort_values('main_publisher', kind='mergesort')  # mergesort is stable
    .reset_index(drop=True)
)
top_paper_publisher_df.head(2)
top_paper_publisher_df.main_publisher.nunique()
path = os.path.join('data', 'scihub-log-publisher-top-articles.tsv.xz')
top_paper_publisher_df.to_csv(path, sep='\t', index=False, compression='xz')<jupyter_output><empty_output><jupyter_text>Quantiles per publisher<jupyter_code>publisher_quantile_df = (
    logs_df
    .merge(journal_df[['scopus_id', 'main_publisher']])
    .groupby('main_publisher')
    .apply(get_quantiles)
    .reset_index(level='main_publisher')
)
publisher_quantile_df.head(2)
path = os.path.join('data', 'scihub-log-publisher-quantiles.tsv.xz')
publisher_quantile_df.to_csv(path, sep='\t', index=False, float_format='%.5g', compression='xz')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15559246954595793, 'fraction_numerical': 0.013104466592838687, 'mean_word_length': 7.004431314623338, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '262148', 'license': 'CC0-1.0', 'n_tokens_mistral': 2149, 'n_tokens_neox': 2039, 'n_words': 289, 'path': '08.log-metrics.ipynb', 'repo_name': 'arielsvn/scihub', 'size': '5418'}