<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "<lambda>() takes 0 positional arguments but 1 was given",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-165-cffa6aedb07e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     83\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 85\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparse_evidence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'evidence.txt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     86\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtypes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# for example\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-165-cffa6aedb07e>\u001b[0m in \u001b[0;36mparse_evidence\u001b[0;34m(file)\u001b[0m\n\u001b[1;32m     72\u001b[0m             \u001b[0;34m'Phospho (STY) Score Diffs'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m             \u001b[0;34m'Phospho (STY) Probabilities'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m             \u001b[0;34m'Oxidation (M) Probabilities'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m     ])\n\u001b[1;32m     76\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-165-cffa6aedb07e>\u001b[0m in \u001b[0;36msiteify\u001b[0;34m(cols)\u001b[0m\n\u001b[1;32m     50\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msiteify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcols\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0;34m\"\"\"Transform column type to SiteString\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m         \u001b[0mconvert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcols\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mSiteString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-165-cffa6aedb07e>\u001b[0m in \u001b[0;36mconvert\u001b[0;34m(cols, fn)\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mcol\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcols\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m                 \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Failed to convert column %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mcol\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-165-cffa6aedb07e>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mcol\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcols\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m                 \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Failed to convert column %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mcol\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: <lambda>() takes 0 positional arguments but 1 was given"
     ]
    }
   ],
   "source": [
    "import re\n",
    "class SiteString:\n",
    "    \"\"\"\n",
    "    A peptide sequence with one or more amino acids mapped to a floating-point score\n",
    "    \"\"\"\n",
    "    \n",
    "    sequence = None\n",
    "    sites = None # Make a dict later\n",
    "    \n",
    "    def __init__(self, string):\n",
    "        \"\"\"\n",
    "        string: A string from the data file like 'AFVNHM(8.97)M(-8.97)SSHSNHPGKR'\n",
    "        \"\"\"\n",
    "        self.sites = {}\n",
    "        pattern = '\\\\((-?[\\\\d.]+)\\\\)'\n",
    "        offset = 0\n",
    "        for match in re.finditer(pattern, string):\n",
    "            self.sites[match.start() + offset] = float(match.group(1))\n",
    "            offset -= len(match.group(0))\n",
    "        self.sequence = re.sub(pattern, '', string)\n",
    "\n",
    "        \n",
    "# Test scored site parser\n",
    "import pytest\n",
    "test_sites = SiteString('AFVNHM(8.97)MM(-8.97)SSHSNH(1.0)PGKR')\n",
    "assert 'AFVNHMMMSSHSNHPGKR' == test_sites.sequence\n",
    "assert {6: 8.97, 8: -8.97, 14: 1.0} == test_sites.sites\n",
    "test_sites_2 = SiteString('')\n",
    "assert '' == test_sites_2.sequence\n",
    "assert {} == test_sites_2.sites\n",
    "\n",
    "\n",
    "def parse_evidence(file):\n",
    "    \n",
    "    def convert(cols, fn):\n",
    "        for col in cols:\n",
    "            try:\n",
    "                df[col] = [fn(s) for s in df[col]]\n",
    "            except ValueError as e:\n",
    "                raise ValueError(\"Failed to convert column %s\" % col) from e\n",
    "    \n",
    "    def split(cols, delim=';'):\n",
    "        \"\"\"Transform column type to list\"\"\"\n",
    "        convert(cols, lambda s: [st.strip() for st in str(s).split(delim)])\n",
    "    \n",
    "    def floatify(cols):\n",
    "        \"\"\"Transform column type to float\"\"\"\n",
    "        convert(cols, lambda: float(s))\n",
    "    \n",
    "    def siteify(cols):\n",
    "        \"\"\"Transform column type to SiteString\"\"\"\n",
    "        convert(cols, lambda: SiteString(str(s)))\n",
    "    \n",
    "    \n",
    "    df = pandas.read_table(file, header=0, index_col=0)\n",
    "\n",
    "    split([\n",
    "            'Protein Group IDs',\n",
    "            'MS/MS IDs',\n",
    "            'Oxidation (M) Site IDs',\n",
    "            'Phospho (STY) Site IDs',\n",
    "            'Modifications',\n",
    "            'Acetyl (Protein N-term)',\n",
    "            'Proteins',\n",
    "            'Leading Proteins',\n",
    "            'Gene Names',\n",
    "            'Protein Names'\n",
    "            ])\n",
    "    split(['Protein Descriptions'], delim='|')\n",
    "    siteify([\n",
    "            'Oxidation (M) Score Diffs',\n",
    "            'Phospho (STY) Score Diffs',\n",
    "            'Phospho (STY) Probabilities',\n",
    "            'Oxidation (M) Probabilities'\n",
    "    ])\n",
    "    \n",
    "    # Some contain \"Infinity\", which float() understands but Pandas doesn't\n",
    "    floatify(['PEP'])\n",<commit_msg>Simplified parse_evidence a bit<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Protein Group IDs                       object\n",
      "Peptide ID                               int64\n",
      "Mod. Peptide ID                          int64\n",
      "MS/MS IDs                               object\n",
      "AIF MS/MS IDs                          float64\n",
      "Oxidation (M) Site IDs                  object\n",
      "Phospho (STY) Site IDs                  object\n",
      "Sequence                                object\n",
      "Length                                   int64\n",
      "Modifications                           object\n",
      "Modified Sequence                       object\n",
      "Oxidation (M) Probabilities             object\n",
      "Phospho (STY) Probabilities             object\n",
      "Oxidation (M) Score Diffs               object\n",
      "Phospho (STY) Score Diffs               object\n",
      "Acetyl (Protein N-term)                 object\n",
      "Oxidation (M)                            int64\n",
      "Phospho (STY)                            int64\n",
      "Proteins                                object\n",
      "Leading Proteins                        object\n",
      "Leading Razor Protein                   object\n",
      "Gene Names                              object\n",
      "Protein Names                           object\n",
      "Protein Descriptions                    object\n",
      "Type                                    object\n",
      "Raw File                                object\n",
      "Experiment                              object\n",
      "Charge                                   int64\n",
      "m/z                                    float64\n",
      "Mass                                   float64\n",
      "Resolution                             float64\n",
      "Uncalibrated - Calibrated m/z [ppm]    float64\n",
      "Mass Error [ppm]                       float64\n",
      "Uncalibrated Mass Error [ppm]          float64\n",
      "Retention Time                         float64\n",
      "Retention Length                       float64\n",
      "Calibrated Retention Time              float64\n",
      "Retention Time Calibration             float64\n",
      "Match Time Difference                  float64\n",
      "PIF                                    float64\n",
      "Fraction of total spectrum             float64\n",
      "Base peak fraction                     float64\n",
      "PEP                                    float64\n",
      "MS/MS Count                              int64\n",
      "MS/MS Scan Number                      float64\n",
      "Score                                  float64\n",
      "Delta score                            float64\n",
      "Combinatorics                            int64\n",
      "Intensity                              float64\n",
      "Reverse                                 object\n",
      "Contaminant                             object\n",
      "dtype: object\n",
      "Protein Group IDs                                                                  [284]\n",
      "Peptide ID                                                                             2\n",
      "Mod. Peptide ID                                                                        2\n",
      "MS/MS IDs                                                                         [2, 3]\n",
      "AIF MS/MS IDs                                                                        NaN\n",
      "Oxidation (M) Site IDs                                                             [nan]\n",
      "Phospho (STY) Site IDs                                                             [nan]\n",
      "Sequence                                                            AADPPAENSSAPEAEQGGAE\n",
      "Length                                                                                20\n",
      "Modifications                                                                         []\n",
      "Modified Sequence                                                 _AADPPAENSSAPEAEQGGAE_\n",
      "Oxidation (M) Probabilities                  <__main__.SiteString object at 0x109c05080>\n",
      "Phospho (STY) Probabilities                  <__main__.SiteString object at 0x109973550>\n",
      "Oxidation (M) Score Diffs                    <__main__.SiteString object at 0x109ec3630>\n",
      "Phospho (STY) Score Diffs                    <__main__.SiteString object at 0x10bca1b38>\n",
      "Acetyl (Protein N-term)                                                              [0]\n",
      "Oxidation (M)                                                                          0\n",
      "Phospho (STY)                                                                          0\n",
      "Proteins                                                                        [P67809]\n",
      "Leading Proteins                                                                [P67809]\n",
      "Leading Razor Protein                                                             P67809\n",
      "Gene Names                                                            [NSEP1, YB1, YBX1]\n",
      "Protein Names                          [CCAAT-binding transcription factor I subunit ...\n",
      "Protein Descriptions                   [>P67809, YBOX1_HUMAN Nuclease-sensitive eleme...\n",
      "Type                                                                          MULTI-MSMS\n",
      "Raw File                                                                   VE20150527-03\n",
      "Experiment                                                                      MAPK1_WT\n",
      "Charge                                                                                 2\n",
      "m/z                                                                             949.4059\n",
      "Mass                                                                            1896.797\n",
      "Resolution                                                                       74589.3\n",
      "Uncalibrated - Calibrated m/z [ppm]                                              -2.8306\n",
      "Mass Error [ppm]                                                                -0.14492\n",
      "Uncalibrated Mass Error [ppm]                                                    -2.9755\n",
      "Retention Time                                                                     25.84\n",
      "Retention Length                                                                 0.96853\n",
      "Calibrated Retention Time                                                          25.84\n",
      "Retention Time Calibration                                                             0\n",
      "Match Time Difference                                                                NaN\n",
      "PIF                                                                                    0\n",
      "Fraction of total spectrum                                                             0\n",
      "Base peak fraction                                                                     0\n",
      "PEP                                                                           2.7775e-55\n",
      "MS/MS Count                                                                            2\n",
      "MS/MS Scan Number                                                                   7282\n",
      "Score                                                                             150.91\n",
      "Delta score                                                                       120.34\n",
      "Combinatorics                                                                          1\n",
      "Intensity                                                                        2106800\n",
      "Reverse                                                                              NaN\n",
      "Contaminant                                                                          NaN\n",
      "Name: 4, dtype: object\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "class SiteString:\n",
    "    \"\"\"\n",
    "    A peptide sequence with one or more amino acids mapped to a floating-point score\n",
    "    \"\"\"\n",
    "    \n",
    "    sequence = None\n",
    "    sites = None # Make a dict later\n",
    "    \n",
    "    def __init__(self, string):\n",
    "        \"\"\"\n",
    "        string: A string from the data file like 'AFVNHM(8.97)M(-8.97)SSHSNHPGKR'\n",
    "        \"\"\"\n",
    "        self.sites = {}\n",
    "        pattern = '\\\\((-?[\\\\d.]+)\\\\)'\n",
    "        offset = 0\n",
    "        for match in re.finditer(pattern, string):\n",
    "            self.sites[match.start() + offset] = float(match.group(1))\n",
    "            offset -= len(match.group(0))\n",
    "        self.sequence = re.sub(pattern, '', string)\n",
    "\n",
    "        \n",
    "# Test scored site parser\n",
    "import pytest\n",
    "test_sites = SiteString('AFVNHM(8.97)MM(-8.97)SSHSNH(1.0)PGKR')\n",
    "assert 'AFVNHMMMSSHSNHPGKR' == test_sites.sequence\n",
    "assert {6: 8.97, 8: -8.97, 14: 1.0} == test_sites.sites\n",
    "test_sites_2 = SiteString('')\n",
    "assert '' == test_sites_2.sequence\n",
    "assert {} == test_sites_2.sites\n",
    "\n",
    "\n",
    "def parse_evidence(file):\n",
    "    \n",
    "    def convert(cols, fn):\n",
    "        for col in cols:\n",
    "            try:\n",
    "                df[col] = [fn(s) for s in df[col]]\n",
    "            except ValueError as e:\n",
    "                raise ValueError(\"Failed to convert column %s\" % col) from e\n",
    "    \n",
    "    def split(cols, delim=';'):\n",
    "        \"\"\"Transform column type to list\"\"\"\n",
    "        convert(cols, lambda s: [st.strip() for st in str(s).split(delim)])\n",
    "    \n",
    "    def floatify(cols):\n",
    "        \"\"\"Transform column type to float\"\"\"\n",
    "        convert(cols, lambda s: float(s))\n",
    "    \n",
    "    def siteify(cols):\n",
    "        \"\"\"Transform column type to SiteString\"\"\"\n",
    "        convert(cols, lambda s: SiteString(str(s)))\n",
    "    \n",
    "    \n",
    "    df = pandas.read_table(file, header=0, index_col=0)\n",
    "\n",
    "    # These are lists\n",
    "    split([\n",
    "            'Protein Group IDs',\n",
    "            'MS/MS IDs',\n",
    "            'Oxidation (M) Site IDs',\n",
    "            'Phospho (STY) Site IDs',\n",
    "            'Modifications',\n",
    "            'Acetyl (Protein N-term)',\n",
    "            'Proteins',\n",
    "            'Leading Proteins',\n",
    "            'Gene Names',\n",
    "            'Protein Names'\n",
    "            ])\n",
    "    \n",
    "    split(['Protein Descriptions'], delim='|')\n",
    "    \n",
    "    # These have syntax like 'AFVNHM(8.97)M(-8.97)SSHSNHPGKR'\n",
    "    siteify([\n",
    "            'Oxidation (M) Score Diffs',\n",
    "            'Phospho (STY) Score Diffs',\n",
    "            'Phospho (STY) Probabilities',\n",
    "            'Oxidation (M) Probabilities'\n",
    "    ])\n",
    "    \n",
    "    # Some contain \"Infinity\", which float() understands but Pandas doesn't\n",
    "    floatify(['PEP'])\n",
{'fraction_non_alphanumeric': 0.16012640316951232, 'fraction_numerical': 0.1112159230261296, 'mean_word_length': 1.2706146926536732, 'pattern_counts': {'":': 33, '<': 16, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 135, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7102130', 'n_tokens_mistral': 9190, 'n_tokens_neox': 7724, 'n_words': 1030}