<jupyter_start><jupyter_text>Gene Conversion AppyterHere we convert a transcript expression count matrix into a gene expression count matrix leveraging official NCBI Symbols.See:  for more information.<jupyter_code>import numpy as np
import pandas as pd
%%appyter hide_code

{% do SectionField(
    name='primary',
    title='Gene Conversion',
    subtitle='Configure your gene conversion',
    img='icon.png',
) %}<jupyter_output><empty_output><jupyter_text>Step 1: Load the input dataIt's important to note that the **first column** of your matrix should contain the symbols we're converting, and the **first row** the column header. More advanced tabular formats are not yet supported.<jupyter_code>%%appyter hide_code

{% do DescriptionField(
    name='desc',
    text='''
      <div class="alert alert-info mb-0">
      Your input matrix should have the gene identifiers to be
      converted in the <b>first column</b> of the matrix, and
      at most <b>one row of header</b>. In the case of Excel which
      is, in general, not recommended due to its tendency to convert
      gene names into dates, only the first sheet will be used.
      </div>
    ''',
    section='primary',
) %}

{% set file = FileField(
  name='file',
  label='File to convert',
  description='Tab-separated matrix to be converted from transcripts to genes',
  default='GSE152418.txt.gz',
  examples={
    'GSE152418.txt.gz': 'https://www.ncbi.nlm.nih.gov/geo/download/?acc=GSE152418&format=file&file=GSE152418%5Fp20047%5FStudy1%5FRawCounts%2Etxt%2Egz',
  },
  section='primary',
) %}

{% set file_format = ChoiceField(
  name='file_format',
  label='File Format',
  description='Please select your file format',
  default='TSV (.tsv / .txt)',
  choices={
    'TSV (.tsv / .txt)': "sep='\\t',",
    'GZipped TSV (.tsv.gz / .txt.gz)': "sep='\\t', compression='gzip',",
    'CSV (.csv)': "sep=',',",
    'GZipped CSV (.csv.gz)': "compression='gzip',",
    'Excel Sheet 1 (.xls, .xlsx, .xlsm, .xlsb, .odf, .ods, .odt)': "excel",
  },
  section='primary',
) %}
%%appyter code_eval

{% if file_format.value == "excel" %}
data = pd.read_excel(
    {{ file }},
    index_col=0,
)
{% else %}
data = pd.read_csv(
    {{ file }},
    index_col=0,
    {{ file_format }}
)
{% endif %}
data
%%appyter hide_code

{% set organism = ChoiceField(
  name='organism',
  label='Select the organism',
  description='Different organisms have different sets of genes available; all NCBI supported organisms can be supported.',
  default='Homo sapiens',
  choices={
    'Homo sapiens': 'ftp://ftp.ncbi.nih.gov/gene/DATA/GENE_INFO/Mammalia/Homo_sapiens.gene_info.gz',
    'Mus musculus': 'ftp://ftp.ncbi.nih.gov/gene/DATA/GENE_INFO/Mammalia/Mus_musculus.gene_info.gz',
    'Rattus norvegicus': 'ftp://ftp.ncbi.nih.gov/gene/DATA/GENE_INFO/Mammalia/Rattus_norvegicus.gene_info.gz',
    'Pan troglodytes': 'ftp://ftp.ncbi.nih.gov/gene/DATA/GENE_INFO/Mammalia/Pan_troglodytes.gene_info.gz',
    'Sus scrofa': 'ftp://ftp.ncbi.nih.gov/gene/DATA/GENE_INFO/Mammalia/Sus_scrofa.gene_info.gz',
    'Canis familiaris': 'ftp://ftp.ncbi.nih.gov/gene/DATA/GENE_INFO/Mammalia/Canis_familiaris.gene_info.gz',
    'Bos taurus': 'ftp://ftp.ncbi.nih.gov/gene/DATA/GENE_INFO/Mammalia/Bos_taurus.gene_info.gz',
    'Mesocricetus auratus': 'https://appyters.maayanlab.cloud/storage/Gene_Conversion_Appyter/Mammalia/Mesocricetus_auratus.gene_info.gz',
  },
  section='primary',
) %}

{% set gene_types = MultiChoiceField(
  name='gene_types',
  label='Gene types',
  description='Types of genes to include in the mapping',
  default=['protein-coding'],
  choices=[
    'protein-coding',
    'ncRNA',
    'pseudo',
    'biological-region',
    'unknown',
    'other',
    'tRNA',
    'snoRNA',
    'snRNA',
    'rRNA',
    'scRNA',
  ],
  section='primary',
) %}
%%appyter code_eval

ncbi_gene_info = pd.read_csv(
    '{{ organism }}',
    compression='gzip', sep='\t',
)
ncbi_gene_info = ncbi_gene_info[np.in1d(ncbi_gene_info['type_of_gene'], {{ gene_types }})]
ncbi_gene_info<jupyter_output><empty_output><jupyter_text>Step 1. Prepare NCBI Gene Synonym lookup tableTo perform this conversion, we'll prepare a lookup table with the ncbi gene info table. Then we'll use that lookup for the conversion.<jupyter_code># create a lookup dictionary using the ncbi table

def maybe_split(record):
  ''' NCBI Stores Nulls as '-' and lists '|' delimited
  '''
  if record in {'', '-'}:
    return set()
  return set(record.split('|'))

def supplement_dbXref_prefix_omitted(ids):
  ''' NCBI Stores external IDS with Foreign:ID while most datasets just use the ID
  '''
  for id in ids:
    # add original id
    yield id
    # also add id *without* prefix
    if ':' in id:
      yield id.split(':', maxsplit=1)[1]

synonym, symbol = zip(*{
  (alternate_symbol, gene_info['Symbol'])
  for _, gene_info in ncbi_gene_info.iterrows()
  for alternate_symbol in set.union(
    maybe_split(gene_info['Symbol']),
    maybe_split(gene_info['Symbol_from_nomenclature_authority']),
    maybe_split(str(gene_info['GeneID'])),
    maybe_split(gene_info['Synonyms']),
    maybe_split(gene_info['Other_designations']),
    maybe_split(gene_info['LocusTag']),
    set(supplement_dbXref_prefix_omitted(maybe_split(gene_info['dbXrefs']))),
  )
})
ncbi_lookup = pd.Series(symbol, index=synonym)
ncbi_lookup
# turns out some synonyms are ambiguous
index_values = ncbi_lookup.index.value_counts()
index_values
# for our sanity we'll drop these entirely, in the future we could consider a better way to take care of these
ncbi_lookup_disambiguated = ncbi_lookup.drop(index_values[index_values > 1].index)
ncbi_lookup_disambiguated<jupyter_output><empty_output><jupyter_text>Step 2: Use Disambiguated NCBI lookup table to map our data<jupyter_code>%%appyter hide_code

{% set versioned_ensembl_id = BoolField(
  name='versioned_ensembl_id',
  label='Do you have versioned ENSEMBL IDs?',
  description='ENSEMBL Ids may be postfixed with the version e.g `ENSG00000227232.1`. If your ENSEMBL ids are versioned, you can check this to prune the versions for a successful conversion.',
  default=False,
  section='primary',
) %}
%%appyter code_eval

# Mapping between current transcripts to the official NCBI Gene symbols, dropping anything that doesn't map
ncbi_lookup_disambiguated_dict = ncbi_lookup_disambiguated.to_dict()
transcript_genes = pd.Series(
{%- if versioned_ensembl_id.value %}
    data.index.map(lambda i: ncbi_lookup_disambiguated_dict.get(i.split('.')[0])),
{%- else %}
    data.index.map(ncbi_lookup_disambiguated_dict.get),
{%- endif %}
    index=data.index,
).dropna()
transcript_genes
# Compute variance of each transcript
transcript_variance = data.var(axis=1).fillna(0)
transcript_variance
# Merge the transcript_genes & transcript_variance
transcript_gene_variance = pd.merge(
    left=transcript_variance.to_frame('variance'), left_index=True,
    right=transcript_genes.to_frame('gene'), right_index=True,
)
transcript_gene_variance
# for all transcripts that map to multiple genes, take the gene, transcript mapping with the highest variance
transcript_gene_mapping = transcript_gene_variance.groupby('gene')['variance'].idxmax()
transcript_gene_mapping
# actually perform the mapping
data_genes = pd.merge(
    left=transcript_gene_mapping.to_frame('transcript'), left_on='transcript',
    right=data, right_index=True,
).drop('transcript', axis=1)
data_genes<jupyter_output><empty_output><jupyter_text>Step 3. Export results<jupyter_code>print(f"Converted Matrix of shape {data.shape} => {data_genes.shape}")
%%appyter code_eval

{% if file_format.value == "excel" %}
data_genes.to_excel(
    'converted_' + {{ file }},
)
{% else %}
data_genes.to_csv(
    'converted_' + {{ file }},
    {{ file_format }}
)
{% endif %}
data_genes
%%appyter markdown
# Your results are available for download **[here](./{{ ('converted_' + file.value)|urlencode }})**.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1365878335651954, 'fraction_numerical': 0.007967623624636398, 'mean_word_length': 4.82756079587325, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '386718', 'license': 'Apache-2.0', 'n_tokens_mistral': 2934, 'n_tokens_neox': 2675, 'n_words': 632, 'path': 'appyters/Gene_Conversion_Appyter/Gene_Conversion.ipynb', 'repo_name': 'MaayanLab/jupyter-template-catalog', 'size': '7907'}