<jupyter_start><jupyter_text>**Aims**: - Explore best ways to present the data - Prepare the publication-quality figure for the manuscript<jupyter_code>%run notebook_setup.ipynb
%vault from pubmed_derived_data import literature
duplicated_doi = literature.doi.dropna()[literature.doi.dropna().duplicated()]
with_duplicated_doi = literature[literature.doi.isin(duplicated_doi)]
with_duplicated_doi
# all duplicates are identical:
assert all(with_duplicated_doi.groupby('doi').agg(set).apply(len) == 1)<jupyter_output><empty_output><jupyter_text>Remove all but the first occurrence of duplicates:<jupyter_code>uids_of_all_but_first = with_duplicated_doi.reset_index().groupby('doi').agg(list).uid.apply(lambda x: x[1:]).sum()
uids_of_all_but_first
literature = literature[~literature.index.isin(uids_of_all_but_first)].copy()
from pandas import Series, to_datetime
from search_terms import primary_terms, secondary_terms, descriptive_terms
terms = [*primary_terms, *secondary_terms, *descriptive_terms]
literature = literature.replace({float('nan'): None}).infer_objects()
literature.date = to_datetime(literature.date, utc=True)
%R -i literature -i terms
%%R
library(ComplexUpset)
source('helpers/plots.R')
source('helpers/colors.R')<jupyter_output><empty_output><jupyter_text>Publication figures<jupyter_code>%vault from pubmed_derived_data import predicted_article_types, reliable_article_types
%vault from pubmed_derived_data import code_repositories
from pandas import concat
combined_article_types = concat([
    predicted_article_types,
    reliable_article_types
]).loc[literature.index]
%vault from pubmed_derived_data import omics_features
from helpers.text_processing import prefix_remover
omics_columns = omics_features.columns

omes_or_omics = (
    omics_features[omics_columns[omics_columns.str.startswith('ome_or_omic_')]]
    .rename(columns=prefix_remover('ome_or_omic_'))
)
literature['omic_terms_detected'] = omes_or_omics.sum(axis=1)
%vault from pubmed_derived_data import domain_features
data = (
    literature
    .drop(columns=['full_text', 'abstract'])
    .join(combined_article_types)
    .join(code_repositories)
    .join(omics_features)
    .join(
        domain_features
        [domain_features.columns[domain_features.columns.str.startswith('mentiones_')]]
    )
)
data['is_type_predicted'] = data.index.isin(predicted_article_types.index)
type_names = {
    'is_review': 'Review',
    'is_method': 'Computational method',
    'is_other_research_article': 'Other research article',
    'is_other': 'Other'
}
def get_type(t):
    if len(t) > 1:
        return 'multiple'
    if len(t):
        return type_names[list(t)[0]]
    return 'unknown'
data['chosen_type'] = combined_article_types.apply(lambda x: get_type(set(x[x != False].index)), axis=1)
data['chosen_type'].value_counts()
from pandas import DataFrame
from statistics import mean
from helpers.stats import bootstrap
oo = []

for predicted in [True, False]:
    for type in combined_article_types.columns:
        d = data[(data[type] != False) & (data['is_type_predicted'] == predicted)].dropna(subset=['omic_terms_detected'])
        boot = bootstrap(list(d['omic_terms_detected']), stat=mean)
        oo.append({
            'is_type_predicted': predicted,
            'type': type_names[type],
            'mean_omics_detected': d['omic_terms_detected'].mean(),
            'mean_omics_detected_ci_lower': boot['lower'],
            'mean_omics_detected_ci_upper': boot['upper']
        })

oo = DataFrame(oo)
%%R -i oo -r 110
(
    ggplot(oo, aes(x=type, y=mean_omics_detected, fill=is_type_predicted))
    + geom_col(position='dodge')
    + geom_errorbar(
        aes(ymin=mean_omics_detected_ci_lower, ymax=mean_omics_detected_ci_upper),
        width=.2,
        position=position_dodge(width = 0.9)
    )
    + theme_bw()
    + scale_fill_manual(values=c('FALSE'='darkgreen', 'TRUE'='green'), name='Article type predicted?')
    + xlab('Article type')
    + ylab('Mean number of omics detected')
    + theme(legend.position = 'bottom')
)
%%R -i oo -r 110 -h 160

plot_counts_summary =function(...) {(
    ggplot(oo[oo$is_type_predicted == FALSE, ], aes(x=type, y=mean_omics_detected))
    + geom_col(
        position='dodge', fill='darkorange',
        ...
    )
    + geom_errorbar(
        aes(ymin=mean_omics_detected_ci_lower, ymax=mean_omics_detected_ci_upper),
        width=.3,
        position=position_dodge(width=0.9),
        size=0.3
    )
    + theme_bw()
    + ylab('Mean number of omics detected (95% CI)')
    + theme(legend.position='bottom', axis.title.y=element_blank())
    + coord_flip()
)}
plot_counts_summary(width=0.5)
integ = 'integrated omics*'
multix = 'multi-view omics**'
other = 'other terms***'
data['term_group'] = data.term.replace({    
    'integrated omics': integ,
    'integrative omics': integ,
    'integromics': integ,
    'multi-block omics': multix,
    'multi-modal omics': multix,
    'multi-source omics': multix,
    'multi-view omics': multix,
    'cross-omics': other,
    'pan-omics': other,
    'poly-omics': other,
    'trans-omics': other,
    'multiple': 'multiple terms'
})
%R -i data
%%R -w 700 -h 400 -r 100
(
    ggplot(data, aes(x=year, fill=term))
    + geom_bar()
    + scale_fill_manual(values=terms_colors)
    + theme_bw()
)
literature[literature[list(terms)].sum(axis=1) > 2][['title', 'doi', 'abstract_clean']]
%%R -w 700 -h 400 -r 100 -i integ -i multix
terms_group_colors = c(
    terms_colors,
    c(
        'multi-omics'='#59A14F',
        'multi-view omics**'='#EDC948',
        'integrated omics*'='#4D79D7',
        'multiple terms'='#b3b7b8',
        'other terms***'='#c85200'
    )
)

term_counts = (
    ggplot(data, aes(x=year, fill=term_group))
    + geom_bar()
    + scale_fill_manual(values=terms_group_colors, name='')
    + theme(legend.position = 'bottom')
    + theme_bw()
)
term_counts
%%R -w 800 -h 400 -r 110

term_counts = (
    ggplot(data, aes(x=year, fill=term_group))
    + geom_area(stat='count')
    + scale_fill_manual(values=terms_group_colors, name='')
    + theme(legend.position = 'bottom')
    + theme_bw()
)
term_counts
%vault from pubmed_derived_data import all_articles_by_journal_and_year
total_articles_in_popular_multi_omics_journals = all_articles_by_journal_and_year.groupby('year')['count'].sum()
total_articles_in_popular_multi_omics_journals.head(2)
term_groups_summary = data.assign(one=1).groupby(['year', 'term_group']).one.sum().rename('count').reset_index()
total_by_year = term_groups_summary.groupby('year')['count'].sum()

present = list(term_groups_summary[['term_group', 'year']].apply(tuple, axis=1))
for term in set(term_groups_summary.term_group):
    for year in set(term_groups_summary.year):
        if (term, year) not in present:
            term_groups_summary = term_groups_summary.append({
                'term_group': term,
                'year': year,
                'count': 0
            }, ignore_index=True)
term_groups_summary['count_adjusted'] = term_groups_summary['count'] / term_groups_summary.year.map(total_articles_in_popular_multi_omics_journals)
total_by_year.head(8)
start_with_year = total_by_year[total_by_year <= 3].idxmax() + 1
term_groups_summary = term_groups_summary[term_groups_summary.year >= start_with_year].copy()
start_with_year
term_groups_summary['yearly_fraction'] = term_groups_summary['count'] / term_groups_summary.year.map(term_groups_summary.groupby('year')['count'].sum())
from datetime import date
SEARCH_DATE = date(year=2020, month=7, day=25)
term_groups_summary['complete_year'] = (term_groups_summary.year + 1).replace({2021: SEARCH_DATE.year + SEARCH_DATE.timetuple().tm_yday / 365})
%%R -w 800 -h 400 -r 110 -i term_groups_summary

(
    ggplot(term_groups_summary, aes(x=complete_year, fill=term_group, y=count_adjusted))
    + geom_area(stat='identity')
    + scale_fill_manual(values=terms_group_colors, name='')
    + theme(legend.position = 'bottom')
    + theme_bw()
)
%%R -w 800 -h 400 -r 110 -i term_groups_summary

term_popularity = (
    ggplot(term_groups_summary, aes(x=complete_year, fill=term_group, y=yearly_fraction))
    + geom_area(stat='identity')
    + scale_fill_manual(values=terms_group_colors, name='')
    + theme(legend.position = 'bottom')
    + theme_bw()
)
term_popularity
%%R -w 800 -h 400 -r 110 -i term_groups_summary

term_counts = ggplot(term_groups_summary, aes(x=complete_year, fill=term_group, y=count)) +
    geom_area(stat='identity') +
    scale_fill_manual(values=terms_group_colors, name='') +
    theme(legend.position = 'bottom') +
    theme_bw() +
    ylab('Articles in PubMed')
term_counts
from repository_detection import (
    source_code_platforms, mixed_publication_platforms, data_only_platforms,
    all_platforms as platforms
)
mostly_code_platforms = list(source_code_platforms) + ['bioconductor']
mostly_data_platforms = list(set(mixed_publication_platforms) - {'bioconductor'}) + list(data_only_platforms)
platform_id_to_name = {
    '.git': 'Other git',
    'bioconductor': 'Bioconductor',
    'bitbucket': 'Bitbucket',
    'cran': 'CRAN',
    'dryad': 'Dryad',
    'github': 'GitHub',
    'gitlab': 'GitLab',
    'osf': 'OSF',
    'pypi': 'PyPI',
    'sourceforge': 'SourceForge',
    'zenodo': 'Zenodo'
}
from pandas import Categorical
platforms_counts = code_repositories[[f'mentions_{platform}' for platform in platforms]].sum().to_frame('count').sort_values('count')
platforms_counts['platform_id'] = platforms_counts.index.str[len('mentions_'):]
platforms_counts['platform_type'] = platforms_counts.platform_id.apply(lambda p: 'code' if p in mostly_code_platforms else 'data')
platforms_counts.loc['mentions_zenodo', 'platform_type'] = 'both'
platforms_counts['platform_type'] = Categorical(platforms_counts['platform_type'], ordered=True, categories=['code', 'data', 'both'])
platforms_counts['platform_name'] = platforms_counts.platform_id.map(platform_id_to_name)
platforms_counts['platform_name'] = Categorical(platforms_counts['platform_name'], ordered=True, categories=reversed(platforms_counts['platform_name']))
platforms_counts['nudge_y'] = 0
platforms_counts.loc['mentions_gitlab', 'nudge_y'] = -0.5
platforms_counts.loc['mentions_osf', 'nudge_y'] = 0.5
platforms_counts['label_top'] = 0
for i, platform in enumerate(platforms_counts[platforms_counts['nudge_y'] == 0].index):
    platforms_counts.loc[platform, 'label_top'] = (((i % 2) - 0.5) * 2) * -1
platforms_counts
%%R -i platforms_counts -r 110 -w 800 -h 190

code_and_data_platforms = function(text_size=3, point_size=3, scale_name='Used for') {
    (
    ggplot(platforms_counts, aes(y=nudge_y, x=count))
    + geom_text(
        data=platforms_counts[platforms_counts$label_top == 0, ],
        aes(
            label=platform_name,
            x=count - 0.5
        ),
        hjust=1,
        size=text_size
    )
    + ggrepel::geom_text_repel(
        data=platforms_counts[platforms_counts$label_top != 0, ],
        aes(
            label=platform_name,
            nudge_y=label_top * 1.3
        ),
        min.segment.length=0,
        segment.curvature=-1e-20,
        segment.color='grey40',
        point.padding=2.5,
        size=text_size
    )
    + geom_point(
        aes(group=platform_name, fill=platform_type, shape=platform_type),
        color='black',
        size=point_size
    )
    + scale_fill_manual(
        values=c(
            'code'='#d62726',
            'data'='#1f77b4',
            'both'='#9467bd'
        ),
        name=scale_name
    )
    + scale_shape_manual(
        values=c(
            'code'='circle filled',
            'data'='square filled',
            'both'='diamond filled'
        ),
        name=scale_name
    )
    + theme_bw()
    + scale_x_log10(
        sec.axis = dup_axis(breaks=platforms_counts$count, name=element_blank()),
        name='Detected use of code and data versioning/distribution platforms'
    )
    #+ annotation_logticks(sides='b')
    + scale_y_continuous(expand=c(1, 0.5, 1, 0.5)) # bottom mult, bottom add, top mult, top add
    + theme(
        legend.position='right',
        axis.title.y=element_blank(),
        axis.ticks.y=element_blank(),
        axis.text.y=element_blank(),
        panel.grid.major.y=element_blank(),
        panel.grid.minor.y=element_blank(),
    )
    )
}

code_and_data_platforms()
20 / len(data) * 100
from helpers.features import number_of_articles_mentioning_feature, eval_features_frame
domain_features_py = eval_features_frame(domain_features)
from pandas import concat
top_n = 10
features_counts = concat([
    number_of_articles_mentioning_feature(
        domain_features_py,
        Series(['diseases', 'clinical_findings']),
        exclude=['disease']
    ).head(top_n),
    number_of_articles_mentioning_feature(
        domain_features_py,
        Series(['species'])
    ).head(top_n)
]).reset_index()
features_counts.head(2)
%%R -i features_counts -r 110
plot_features = function(names_padding=0.05, counts_padding=0.05, expand=expansion(mult=0, add=0)) {
    (
    ggplot(features_counts, aes(x=kind, y=rank))
    + geom_tile(fill='white')
    + geom_text(aes(label=strtrim(term, 25)), hjust=0, position=position_nudge(x=-(0.5-names_padding)), size=3)
    + geom_text(
        aes(label=count, color=proportion_of_articles),
        hjust=1, position=position_nudge(x=0.5-counts_padding), size=3.4
    )
    + theme_bw()
    + scale_y_reverse()
    + scale_x_discrete(
        position="top",
        labels=c(
            'diseases,clinical_findings'='disease/clinical finding',
            'species'='species'
        ),
        name='Top diseases and species mentioned in the abstracts',
        expand=expand
    )
    + theme(
        legend.position='bottom',
        axis.title.y=element_blank(),
        axis.ticks.y=element_blank(),
        axis.text.y=element_blank(),
    )
)}
plot_features(names_padding=0.05, counts_padding=0.05)
from omics import omics_by_entity, omics_by_entity_group
omic_entities_columns = Series(list(omics_by_entity.keys()))
omic_entities_group_columns = Series(list(omics_by_entity_group.keys()))
omic_entities = data.rename(columns=prefix_remover('entity_', enforce=False))
omic_entities_groups = data.rename(columns=prefix_remover('entity_group_', enforce=False))
%R -i omic_entities
%R -i omic_entities_groups
%%R -w 800 -r 100 -i omic_entities_columns

omics_upset = function(...) {
    upset(
        omic_entities,
        omic_entities_columns,
        min_size=50,
        width_ratio=0.2,
        min_degree=3,
        base_annotations=list(
            'Intersection size'=intersection_size(
                counts=FALSE,
                mapping=aes(fill=term_group)
            )
        ),
        themes=upset_modify_themes(
            list(
                'overall_sizes'=theme(axis.text.x=element_text(angle=90))
            )
        ),
        labeller=function(x) {
            c(
                'endogenous molecules'='metabolites²',
                'genes'='genes',
                'transcripts'='transcripts',
                'proteins & peptides'='proteins¹',
                'epigenetic modifications'='epigenetic³'
            )[x]
        },
        set_sizes=FALSE,
        ...
    ) + theme(axis.title.x=element_blank()) & scale_fill_manual(values=terms_group_colors, name='')
}
omics_upset()
%%R -r 110 -w 800 -h 800

upset_plot = wrap_elements(
    omics_upset(matrix=intersection_matrix(geom=geom_point(size=1.75)))
    & theme(
        legend.position='none',
        plot.margin=unit(c(0, 0, -0.3, 0), "cm"),
        panel.background=element_rect(fill='transparent', colour=NA),
        plot.background=element_rect(fill='transparent', colour=NA)
    )
    + theme(panel.border=element_rect(color='grey20', fill=NA))
)

TITLE_SIZE = 10
common_theme = theme(
    axis.title.x=element_blank(),
    plot.title=element_text(size=TITLE_SIZE)
)
%%R -r 110 -w 800 -h 800
figure = (
    (
        (
            (
                (
                    upset_plot
                    + labs(
                        title='A. Frequently discussed omics combinations'
                    )
                    + common_theme
                )
                |
                (
                    term_counts
                    + common_theme
                    + labs(
                        title='B. Multi-omics articles indexed by PubMed'
                    )
                )
            )
            & theme(legend.position='bottom')
            & plot_layout(guides='collect')
        )
        /
        (
            (
                plot_counts_summary(width=0.8)
                + scale_y_continuous(
                    expand=c(0, 0.02, 0, 0.1),
                    position='right',
                )
                + shadowtext::geom_shadowtext(
                    aes(label=type, y=0.1), hjust=0, size=3,
                    bg.r=0.05, color='black', bg.color='orange'
                )
                + theme(
                    legend.position='right',
                    axis.title.y=element_blank(),
                    axis.ticks.y=element_blank(),
                    axis.text.y=element_blank(),
                    axis.title.x=element_blank(),
                    plot.title.position='plot'
                )
                + common_theme
                + labs(title='C. Mean number of omics detected (95% CI)')
            )
            |
            (
                plot_features(names_padding=0.035, counts_padding=0.035, expand=expansion(add=0.0))
                + guides(
                    color=guide_colorbar(
                        barheight=0.5,
                        label.position='top',
                        title.vjust=0.5,
                        title.theme=element_text(size=9),
                        order=0,
                        #label.vjust=-5,
                        label.hjust=0
                    )
                )
                + common_theme
                + scale_color_gradient(
                    low='grey40',
                    high='red',
                    name='D) Proportion of publications',
                    breaks=c(0.0, 0.1, 0.2),
                    limits=function(automatic_limits) {
                        automatic_limits[[1]] = 0
                        automatic_limits
                    },
                    labels=scales::percent
                )
                + labs(
                    title='D. Top diseases & species mentioned in the abstracts'
                )
            ) & theme(legend.position='bottom')
        )
        /
        (
            code_and_data_platforms(scale_name='E) Platform used for')
            + common_theme
            + labs(
                title='E. Detected use of code and data versioning/distribution platforms'
            )
            + guides(
                fill=guide_legend(
                    keywidth=0.1,
                    title.theme=element_text(size=9),
                )
            )
        ) & theme(
            legend.position='bottom',
            legend.box='horizontal',
            legend.direction='horizontal',
            legend.margin=unit(-15, 'cm'),
        )
    )
    + plot_layout(
        heights=c(3, 2, 0.9),
        guides='collect'
    ) + plot_annotation(title='Multi-omics literature overview')
)
figure
%R ggsave(filename='figures/overview.pdf', plot=figure, scale=1.1)
%R ggsave(filename='figures/overview.tiff', plot=figure, scale=1.1, dpi=300)
%R ggsave(filename='figures/overview.png', plot=figure, scale=1.1, device='png', dpi=105)<jupyter_output>R[write to console]: Saving 7.33 x 7.33 in image

R[write to console]: Saving 7.33 x 7.33 in image

R[write to console]: Saving 7.33 x 7.33 in image<jupyter_text>Compress the PNG if trimage is available:<jupyter_code>from shutil import which
from contextlib import redirect_stdout
from io import StringIO

if which('trimage'):
    f = StringIO()
    with redirect_stdout(f):
        !trimage -f figures/overview.png<jupyter_output><empty_output><jupyter_text>Details for captions Proportion full-text articles among all articles:<jupyter_code>(data.has_full_text==True).mean()<jupyter_output><empty_output><jupyter_text>Proportion full-text articles among PMC articles (this is because non-PMC are NA, and are not accounted for when calculating the mean below!):<jupyter_code>data.has_full_text.mean()<jupyter_output><empty_output><jupyter_text>Alternative UpSet plots<jupyter_code>%%R -w 800 -r 100 -i omic_entities_group_columns

upset(omic_entities_groups, omic_entities_group_columns, min_size=50, width_ratio=0.1, min_degree=3)
%%R -w 800 -r 100 -i omic_entities_columns

upset(
    omic_entities,
    omic_entities_columns,
    min_size=50,
    width_ratio=0.2,
    min_degree=3,
    base_annotations=list(
        'Intersection size'=intersection_size(
            counts=FALSE,
            aes=aes(fill=chosen_type)
        )
    ),
    themes=upset_modify_themes(
        list(
            'overall_sizes'=theme(axis.text.x=element_text(angle=90))
        )
    ),
    set_sizes=FALSE
)<jupyter_output>[1] "Dropping empty groups: clades (meta_* & pan_*), protein modifications, exogenous factors, clinical data"
{'fraction_non_alphanumeric': 0.1270170803057469, 'fraction_numerical': 0.017552137397376617, 'mean_word_length': 3.002832861189802, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '26113', 'license': 'MIT', 'n_tokens_mistral': 7432, 'n_tokens_neox': 6895, 'n_words': 1153, 'path': 'notebooks/Overview.ipynb', 'repo_name': 'NehaKoppikar/multi-omics-state-of-the-field', 'size': '21194'}