<jupyter_start><jupyter_text>Coverage comparisonsAn unfortunate amount of code overlap with `10.state-of-OA.ipynb`.<jupyter_code>import collections
import itertools
import json
import pathlib

import pandas
from statsmodels.stats.proportion import proportion_confint<jupyter_output><empty_output><jupyter_text>Integrate coverage datasets with library access accuracy analysis on 500 DOIs<jupyter_code>with pathlib.Path('00.configuration.json').open() as read_file:
    config = json.load(read_file)
path = pathlib.Path('data/doi.tsv.xz')
article_df = pandas.read_table(path, compression='xz')
article_df.head(2)
path = pathlib.Path('data/state-of-oa-dois.tsv.xz')
oadoi_df = pandas.read_table(path, compression='xz')
oadoi_df['oadoi'] = (oadoi_df.oadoi_color != 'closed').astype(int)
oadoi_df.head(2)
url = config['library_access_url'] + 'evaluate_library_access_from_output_tsv/manual-doi-checks-500.tsv'
penn_df = (
    pandas.read_table(url)
    .drop(['penn_access_date', 'open_access_date'], axis='columns')
    .rename(columns={'penn_access': 'penn', 'open_access': 'open'})
)
penn_df.head(2)
access_df = (
    article_df
    .rename(columns={'in_scihub_dois': 'scihub'})
    [['doi', 'scihub']]
    .merge(penn_df)
    .merge(oadoi_df)
)
access_df.head(3)
repositories = ['open', 'oadoi', 'penn', 'penntext', 'scihub']
access_df[repositories].mean()
path = pathlib.Path('data/library-access-dois-500.tsv')
access_df.to_csv(str(path), sep='\t', index=False)<jupyter_output><empty_output><jupyter_text>Compute coverage of repo combos<jupyter_code>path = pathlib.Path('data/library-access-dois-500.tsv')
access_df = pandas.read_table(path)
# Read State of OA coverage that's already computed
oadoi_cov_df = pandas.read_table('data/state-of-oa-coverage.tsv', dtype={'venn': str})
oadoi_cov_df.head(2)
def get_combinations(repositories):
    """
    Find all possible repository combinations
    """
    combinations = list()
    for i in range(len(repositories)):
        combinations.extend(itertools.combinations(repositories, i + 1))
    return list(map(list, combinations))

renamer = {
    'open': 'None',
    'oadoi': 'oaDOI',
    'penn': 'Penn',
    'penntext': 'PennText',
    'scihub': 'Sci-Hub',
}

def summarize(df, repositories):
    """
    Compute coverage for all repo combinations
    """
    rows = list()
    combinations = get_combinations(repositories)
    for repos in combinations:
        row = collections.OrderedDict()
        row['venn'] = ''.join('1' if repo in repos else '0' for repo in repositories)
        row['repos'] = ', '.join(renamer[repo] for repo in repos)
        row['n_repos'] = len(repos)
        access = df[repos].max(axis='columns')
        row['available'] = sum(access)
        row['articles'] = len(access)
        row['coverage'] = access.mean()
        row['coverage_lower'], row['coverage_upper'] = proportion_confint(
            count=row['available'], nobs=row['articles'], alpha=0.05, method='jeffreys')
        row['coverage_formatted'] = '{coverage:.1%} [{coverage_lower:.1%}–{coverage_upper:.1%}]'.format(**row)
        rows.append(row)
    return pandas.DataFrame(rows)
repositories = 'open', 'penn', 'scihub'
all_df = summarize(access_df, repositories)
all_df
# Max margin of error
max(list(all_df.coverage - all_df.coverage_lower) + list(all_df.coverage_upper - all_df.coverage))
repositories = 'oadoi', 'penn', 'scihub'
toll_df = summarize(access_df.query("open == 0"), repositories)
toll_df
# Max margin of error
max(list(toll_df.coverage - toll_df.coverage_lower) + list(toll_df.coverage_upper - toll_df.coverage))<jupyter_output><empty_output><jupyter_text>Venn diagramsAn alternative [may be](https://github.com/hms-dbmi/UpSetR/issues/102issuecomment-351193870) UpSetR-style axis labels.<jupyter_code>import matplotlib
import matplotlib_venn

%matplotlib inline

matplotlib.pyplot.style.use('ggplot')

# Fonts: http://jonathansoma.com/lede/data-studio/matplotlib/list-all-fonts-available-in-matplotlib-plus-samples/
matplotlib.rcParams['font.sans-serif'] = "Garuda"
matplotlib.rcParams['font.family'] = "sans-serif"
# matplotlib.rcParams['figure.figsize'] = 2.5, 2.5
cmap = matplotlib.cm.get_cmap('viridis')
# cmap = matplotlib.colors.LinearSegmentedColormap.from_list(
#     name='coverage_cmap', colors=['#ffffff', '#000000'])

def venn_plotter(df, repositories, ax=None):
    """
    Use matplotlib-venn
    https://github.com/konstantint/matplotlib-venn
    """
    labels = [renamer[repo] for repo in repositories]
    venn = matplotlib_venn.venn3_unweighted(
        subsets=7 * [1],
        set_labels=labels,
        alpha=1,
        ax=ax,
    )
    circles = matplotlib_venn.venn3_circles(
        subsets=7 * [1],
        linestyle='dotted',
        linewidth=1.5,
        color='#ffffff',
        ax=ax,
    )

    for row in df.itertuples():
        label = venn.get_label_by_id(row.venn)
        label.set_text(f'{100 * row.coverage:.1f}')
        label.set_fontsize(9 if row.n_repos == 2 else 11)
        label.set_color('#000000' if row.coverage > 0.75 else '#ffffff')
        patch = venn.get_patch_by_id(row.venn)
        patch.set_color(cmap(row.coverage))

    if not ax:
        figure = matplotlib.pyplot.figure()
        return figure
def set_title(axes, label, rightside=False):
    """
    Set title of subplot on top (rightside=False) or on right (rightside=True).
    Uses workaround from https://stackoverflow.com/a/47777701/4651668
    """
    if rightside:
        text = axes.text(s=label, x=0.8, y=-0.05, verticalalignment='center', rotation=270, size=12.5)
    else:
        text = axes.set_title(label, loc='center', y=1.05, size=12.5)
    text.set_bbox({'facecolor': '#FEF2E2', 'boxstyle': 'round', 'edgecolor': '#ccb494'})
    return text
figure, axes = matplotlib.pyplot.subplots(nrows=2, ncols=2, figsize=(2 * 2.9, 2 * 2.9))

# Panel 1
repositories = 'open', 'penn', 'scihub'
n_articles, = all_df.articles.unique()
venn_plotter(all_df, repositories, ax=axes[0, 0])
text = set_title(axes[0, 0], f'{n_articles:,} articles', rightside=False)

# Panel 2
repositories = 'oadoi', 'penn', 'scihub'
n_articles, = toll_df.articles.unique()
venn_plotter(toll_df, repositories, ax=axes[0, 1])
text = set_title(axes[0, 1], f'{n_articles:,} toll articles', rightside=False)
# text = set_title(axes[0, 1], 'Manual', rightside=True)

# Panel 3
repositories = 'oadoi', 'penntext', 'scihub'
venn_df = oadoi_cov_df.query("oadoi_color == 'all' and collection == 'Combined'")
n_articles, = venn_df.articles.unique()
venn_plotter(venn_df, repositories, ax=axes[1, 0])
text = set_title(axes[1, 0], f'{n_articles:,} articles', rightside=False)

# Panel 4
venn_df = oadoi_cov_df.query("oadoi_color == 'closed + green' and collection == 'Combined'")
venn_plotter(venn_df, repositories, ax=axes[1, 1])
n_articles, = venn_df.articles.unique()
text = set_title(axes[1, 1], f'{n_articles:,} toll articles', rightside=False)
# text = set_title(axes[1, 1], 'Automatic', rightside=True)

matplotlib.pyplot.subplots_adjust(wspace=0.1, hspace=0.3)
matplotlib.pyplot.savefig('figure/library-access-venns-small.svg', bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15562403697996918, 'fraction_numerical': 0.021291497408600646, 'mean_word_length': 4.739549839228296, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '379335', 'license': 'CC0-1.0', 'n_tokens_mistral': 2804, 'n_tokens_neox': 2600, 'n_words': 463, 'path': '14.coverage-venns.ipynb', 'repo_name': 'arielsvn/scihub', 'size': '7139'}