<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# INSTALL\n",
    "- WARNING ::: ONLY DO ONCE \n",
    "    - update devconfig in ~/.config/pyontutils/devonfig.yaml\n",
    "        - scigraph_api: http://scigraph.scicrunch.io:9000/scigraph\n",
    "    - Install both pyontutils and ilxutils with pyontutils\n",
    "        - cd ~/git/pyontutils\n",
    "        - pip3 install --user --editable .\n",
    "        - cd ~/git/pyontutils/ilxutils/\n",
    "        - pip3 install --user --editable .\n",
    "    - Clone ontquery and install\n",
    "        - cd ~/git\n",
    "        - git clone https://github.com/tgbugs/ontquery.git\n",
    "        - cd ~/git/ontquery\n",
    "        - pip3 install --user --editable ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Maintainance\n",
    "- update repos\n",
    "    - cd ~/git/pyontutils\n",
    "    - git pull \n",
    "    - cd ~/git/ontquery\n",
    "    - git pull"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Google Sheets Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>curie</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>abdominal cavity</td>\n",
       "      <td>UBERON:0003684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>abdominal wall</td>\n",
       "      <td>UBERON:0003697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>adipose tissue</td>\n",
       "      <td>UBERON:0001013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>adult organism</td>\n",
       "      <td>UBERON:0007023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>alimentary part of gastrointestinal system</td>\n",
       "      <td>UBERON:0005409</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "0                                        term           curie\n",
       "1                            abdominal cavity  UBERON:0003684\n",
       "2                              abdominal wall  UBERON:0003697\n",
       "3                              adipose tissue  UBERON:0001013\n",
       "4                              adult organism  UBERON:0007023\n",
       "5  alimentary part of gastrointestinal system  UBERON:0005409"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pyontutils.sheets import Sheet\n",
    "import pandas as pd\n",
    "\n",
    "class Brainstem(Sheet):\n",
    "    name = 'sparc-terms' # key name you gave the google sheet id value in secrets.yaml\n",
    "    sheet_name = 'Minimal information model(MIS)' # the actual sheet name on the google sheet\n",
    "    fetch_grid = True # meta data in self.grid that has detials like bolding\n",
    "    \n",
    "brainstem = Brainstem()\n",
    "df = pd.DataFrame(brainstem.raw_values)\n",
    "df.columns = df.iloc[0]\n",
    "df.drop(df.index[0], inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# QUERY DATABASES "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[OntTerm('HBA:3999', label='brain (hba)'),\n",
       " OntTerm('FMA:50801', label='Brain'),\n",
       " OntTerm('UBERON:0000955', label='brain'),\n",
       " OntTerm('UBERON:6110636', label='adult cerebral ganglion'),\n",
       " OntTerm('ILX:0101431', label='Brain'),\n",
       " OntTerm('ILX:0101433', label='Brain Infarction'),\n",
       " OntTerm('ILX:0506386', label='Brain Aneurysm'),\n",
       " OntTerm('ILX:0433050', label='Brain Chemistry'),\n",
       " OntTerm('ILX:0641746', label='alpha BRAIN'),\n",
       " OntTerm('ILX:0726394', label='brain meninx'),\n",
       " OntTerm('ILX:0729002', label='brain commissure'),\n",
       " OntTerm('ILX:0101434', label='Brain Ischemia'),\n",
       " OntTerm('ILX:0461406', label='Brain Death'),\n",<commit_msg>added more examples<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# INSTALL\n",
    "- WARNING ::: ONLY DO ONCE \n",
    "    - update devconfig in ~/.config/pyontutils/devonfig.yaml\n",
    "        - scigraph_api: http://scigraph.scicrunch.io:9000/scigraph\n",
    "    - Install both pyontutils and ilxutils with pyontutils\n",
    "        - cd ~/git/pyontutils\n",
    "        - pip3 install --user --editable .\n",
    "        - cd ~/git/pyontutils/ilxutils/\n",
    "        - pip3 install --user --editable .\n",
    "    - Clone ontquery and install\n",
    "        - cd ~/git\n",
    "        - git clone https://github.com/tgbugs/ontquery.git\n",
    "        - cd ~/git/ontquery\n",
    "        - pip3 install --user --editable ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Maintainance\n",
    "- update repos\n",
    "    - cd ~/git/pyontutils\n",
    "    - git pull \n",
    "    - cd ~/git/ontquery\n",
    "    - git pull"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Google Sheets Import\n",
    "### Need pyontutils secrets.yaml setup first!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>curie</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>abdominal cavity</td>\n",
       "      <td>UBERON:0003684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>abdominal wall</td>\n",
       "      <td>UBERON:0003697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>adipose tissue</td>\n",
       "      <td>UBERON:0001013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>adult organism</td>\n",
       "      <td>UBERON:0007023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>alimentary part of gastrointestinal system</td>\n",
       "      <td>UBERON:0005409</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "0                                        term           curie\n",
       "1                            abdominal cavity  UBERON:0003684\n",
       "2                              abdominal wall  UBERON:0003697\n",
       "3                              adipose tissue  UBERON:0001013\n",
       "4                              adult organism  UBERON:0007023\n",
       "5  alimentary part of gastrointestinal system  UBERON:0005409"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pyontutils.sheets import Sheet\n",
    "import pandas as pd\n",
    "KEY_NAME = 'sparc-terms'\n",
    "SHEET_NAME = 'Minimal information model(MIS)'\n",
    "\n",
    "class Brainstem(Sheet):\n",
    "    name = KEY_NAME # key name you gave the google sheet id value in secrets.yaml\n",
    "    sheet_name = SHEET_NAME # the actual sheet name on the google sheet\n",
    "    fetch_grid = True # meta data in self.grid that has detials like bolding\n",
    "\n",
    "brainstem = Brainstem()\n",
    "df = pd.DataFrame(brainstem.raw_values)\n",
    "df.columns = df.iloc[0]\n",
    "df.drop(df.index[0], inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abdominal cavity',\n",
       " 'abdominal wall',\n",
       " 'adipose tissue',\n",
       " 'adult organism',\n",
       " 'alimentary part of gastrointestinal system',\n",
       " 'arterial blood',\n",
       " 'biceps femoris',\n",
       " 'blood',\n",
       " 'bolus of food',\n",
       " 'brainstem']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(df.term)[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CSV or TSV  EXAMPLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "\n",
    "csv_df = pd.DataFrame('/path/to/csv')\n",
    "tsv_df = pd.DataFrame('/path/to/tsv', delimiter='\\t')\n",
    "\n",
    "csv_df.head() # returns top 5 rows\n",
    "csv_df.column_name # specific column name will return a Series which will ac"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# QUERY DATABASES "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[OntTerm('HBA:3999', label='brain (hba)'),\n",
       " OntTerm('FMA:50801', label='Brain'),\n",
       " OntTerm('UBERON:0000955', label='brain'),\n",
       " OntTerm('UBERON:6110636', label='adult cerebral ganglion'),\n",
       " OntTerm('ILX:0101431', label='Brain'),\n",
       " OntTerm('ILX:0101433', label='Brain Infarction'),\n",
       " OntTerm('ILX:0506386', label='Brain Aneurysm'),\n",
       " OntTerm('ILX:0433050', label='Brain Chemistry'),\n",
       " OntTerm('ILX:0641746', label='alpha BRAIN'),\n",
       " OntTerm('ILX:0726394', label='brain meninx'),\n",
       " OntTerm('ILX:0729002', label='brain commissure'),\n",
       " OntTerm('ILX:0101434', label='Brain Ischemia'),\n",
       " OntTerm('ILX:0461406', label='Brain Death'),\n",
{'fraction_non_alphanumeric': 0.20704845814977973, 'fraction_numerical': 0.032158590308370046, 'mean_word_length': 1.6872632575757576, 'pattern_counts': {'":': 78, '<': 119, '<?xml version=': 0, '>': 119, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3827590', 'n_tokens_mistral': 4485, 'n_tokens_neox': 4148, 'n_words': 701}