<jupyter_start><jupyter_text>Code archives and repositories extraction<jupyter_code>from re import escape
from pandas import Series

from repository_detection import (
    source_code_platforms, mixed_publication_platforms, data_only_platforms,
    all_platforms as platforms
)
source_code_platforms
mixed_publication_platforms
data_only_platforms
def collapse_lists(lists):
    return sorted(set(sum(lists, [])))


fields = Series(['abstract', 'full_text'])

for platform in platforms:
    for field in fields:
        code_repositories[f'{field}_mentions_{platform}'] = literature[field].str.lower().str.contains(escape(platform)) == True
        matches = literature[field].astype(str).str.findall(platforms[platform])
        code_repositories[f'{field}_{platform}_matches'] = matches
    code_repositories[f'mentions_{platform}'] = code_repositories[fields + f'_mentions_{platform}'].any(axis=1)
    code_repositories[f'{platform}_matches'] = code_repositories[fields + f'_{platform}_matches'].apply(collapse_lists, axis=1)<jupyter_output><empty_output><jupyter_text>Ext-link handling is not needed for abstracts:<jupyter_code>assert not literature.abstract.str.contains('ext-link').any()
code_repositories[[f'abstract_mentions_{platform}' for platform in platforms]].sum()
code_repositories[[f'full_text_mentions_{platform}' for platform in platforms]].sum()<jupyter_output><empty_output><jupyter_text>The sourceforge mentions might be uses of tools.<jupyter_code>code_repositories[[f'mentions_{platform}' for platform in platforms]].sum()
code_repositories[[f'{platform}_matches' for platform in platforms]].sum()
Series(code_repositories['.git_matches'].sum()).sorted_value_counts()<jupyter_output><empty_output><jupyter_text>Limitations - does not account for GitLab instances in custom domains  Screen for code availability statements We can compose a list of common code availability phrases and use that to see what are the other ways of sharing code.<jupyter_code>journals_with_most_papers = literature['journal'].sorted_value_counts().head(20)
journals_with_most_papers.head(10)
literature = literature.drop(columns=list(set(literature.columns) & set(code_repositories.columns)))
likely_to_contain_code = Series(list(source_code_platforms) + list(mixed_publication_platforms))
any_source_link = code_repositories['abstract_' + likely_to_contain_code + '_matches'].any(axis=1)
any_source_link.sum()<jupyter_output><empty_output><jupyter_text>Note: this does not have to their source code - can be of a tool they used!<jupyter_code>literature['has_source_code_link'] = any_source_link
import pandas
pandas.set_option('display.max_colwidth', 500)
from pandas import DataFrame
from helpers.n_grams import find_longest_common_n_grams


statments = []

for journal in journals_with_most_papers.index:
    relevant_papers = literature[(literature.journal == journal) & literature.has_source_code_link]
    relevant_papers_with_abstracts = relevant_papers[~relevant_papers.abstract.isnull()]

    statments.append({
        'journal': journal,
        'papers_with_code_link': len(relevant_papers),
        'common_n_grams': find_longest_common_n_grams(
            data=relevant_papers_with_abstracts.abstract,
            min_words=3, max_words=10,
            min_count=3, min_frequency=0.5
        )
    })

DataFrame(statments)<jupyter_output><empty_output><jupyter_text>Only _Bioinformatics (Oxford, England)_, _Frontiers in genetics_ and _BMC bioinformatics_ had n-grams which looked like code availability statements.<jupyter_code>compact_code_links_summary = (
    code_repositories['abstract_' + Series(list(platforms)) + '_matches']
    .rename(columns=lambda c: c.replace('abstract_', '').replace('_matches', ''))
    .apply(lambda x: x[x.apply(len) != 0].to_dict(), axis=1)
    .to_frame('detected_code_links')
)
def get_statments(expression):
    return (
        literature['abstract'].str.lower()
        .str.extract(expression).dropna()
        .rename(columns={0: 'match'})
        .join(compact_code_links_summary)
        .join(literature)[['match', 'detected_code_links', 'journal']]
        .rename(columns=lambda x: x.replace('abstract_', ''))
    )
pandas.set_option('display.max_colwidth', 100)<jupyter_output><empty_output><jupyter_text>BMC bioinformatics<jupyter_code>get_statments('is freely available (.*?)\n')
get_statments('available at (https.*?)\s')<jupyter_output><empty_output><jupyter_text>Bioinformatics (Oxford)<jupyter_code>header = 'availability and implementation'

get_statments(f'{escape(header)}\n(.*?)\n')<jupyter_output><empty_output><jupyter_text>Store the data<jupyter_code>%vault store code_repositories in pubmed_derived_data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11714770797962648, 'fraction_numerical': 0.00466893039049236, 'mean_word_length': 7.070205479452055, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '610462', 'license': 'MIT', 'n_tokens_mistral': 1548, 'n_tokens_neox': 1466, 'n_words': 254, 'path': 'notebooks/Source_code.ipynb', 'repo_name': 'NehaKoppikar/multi-omics-state-of-the-field', 'size': '4712'}