<commit_before>{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from collections import defaultdict\n",
      "import re"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#interprot_annot= defaultdict(list)\n",
      "interprot_annot = {}\n",
      "GENE_ID = 0\n",
      "ANNOT_TOOL = 1\n",
      "TAGS = 8"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def extract_signature_desc(tag_line):\n",
      "    signature_desc = \"\"\n",
      "    tag_fields = tag_line.split(\";\")\n",
      "    for field in tag_fields:\n",
      "        if \"signature_desc\" in field:\n",
      "            value_start = field.find(\"=\")\n",
      "            signature_desc = field[value_start+1:]\n",
      "    return signature_desc"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Extract the signature description from Interpro annotation using \n",
      "with open(\"TcIV-X10825.v2-AugustusHintsProt-InterproScan.gff3\",\"r\") as interprot_fh:\n",
      "    for line in interprot_fh:\n",
      "        if line[0] != \"#\" : #ignore comment lines\n",
      "            line_fields = line.rstrip(\"\\n\").split(\"\\t\")\n",
      "            \n",
      "            #Extract from Pfam lines the\n",
      "            if line_fields[ANNOT_TOOL].lower() == \"pfam\":\n",
      "                signature_desc = extract_signature_desc(line_fields[TAGS])\n",
      "                #Remove the .t1 to the gene_id\n",
      "                dot_position = line_fields[GENE_ID].rfind(\".\")\n",
      "                if dot_position != -1:\n",
      "                    line_fields[GENE_ID] = line_fields[GENE_ID][:dot_position]\n",
      "                interprot_annot[  line_fields[GENE_ID] ] = signature_desc\n",
      "        elif line.rstrip(\"\\n\") == \"##FASTA\":\n",
      "            break"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def getSignatureDescription(gene_id, signature_descriptions):\n",
      "    return signature_descriptions[gene_id] if gene_id in signature_descriptions else \"putative protein\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Substitutes the orignal ID, or parent ID for the signature_desc from interprot pfam if available\n",
      "def processAugLine(line, signature_descriptions):\n",
      "    id_match = re.search( r\"ID=(.+?)(\\..*?)?(;|$)\", line)\n",
      "    if id_match:\n",
      "        gene_id = id_match.group(1)\n",
      "        new_gene_id = getSignatureDescription(gene_id,signature_descriptions)\n",
      "        \n",
      "        new_line = re.sub(r\"ID=(.+?)(;|\\n)\", \"ID=\"+new_gene_id+r\"\\g<2>\",line)\n",
      "        new_line = re.sub(r\";Parent=(.+?)(;|\\n|$)\", r\"\\g<2>\",new_line)\n",
      "    else:\n",
      "        #Look for parent id \n",
      "        parent_match = re.search( r\"Parent=(.+?)(\\..*?)?(;|$)\", line)\n",
      "        if parent_match:\n",
      "            parent_id = parent_match.group(1)\n",
      "            new_parent_id = getSignatureDescription(parent_id,signature_descriptions)\n",
      "            new_line = re.sub(r\"Parent=(.+?)(;|\\n)\", \"Parent=\"+new_parent_id+r\"\\g<2>\",line)\n",
      "        else:\n",
      "            new_line = line\n",
      "    return new_line"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#putative_prot_counter = 0\n",
      "with open(\"merged.gff\",\"r\") as aug_fh , open(\"annotated_merged.gff\",\"w\") as out_fh:\n",
      "    for line in aug_fh:\n",
      "        if line[0] == \"#\":\n",
      "            out_fh.write(line)\n",
      "        else:\n",
      "            out_fh.write(processAugLine(line, interprot_annot))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    }
   ],
   "metadata": {}
  }
 ]
}<commit_msg>Modified annotation to fit format expected by augustus tools<commit_after>{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from collections import defaultdict\n",
      "import re"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#interprot_annot= defaultdict(list)\n",
      "interprot_annot = {}\n",
      "GENE_ID = 0\n",
      "ANNOT_TOOL = 1\n",
      "TAGS = 8"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def extract_signature_desc(tag_line):\n",
      "    signature_desc = \"\"\n",
      "    tag_fields = tag_line.split(\";\")\n",
      "    for field in tag_fields:\n",
      "        if \"signature_desc\" in field:\n",
      "            value_start = field.find(\"=\")\n",
      "            signature_desc = field[value_start+1:]\n",
      "    return signature_desc"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Extract the signature description from Interpro annotation using \n",
      "with open(\"TcIV-X10825.v2-AugustusHintsProt-InterproScan.gff3\",\"r\") as interprot_fh:\n",
      "    for line in interprot_fh:\n",
      "        if line[0] != \"#\" : #ignore comment lines\n",
      "            line_fields = line.rstrip(\"\\n\").split(\"\\t\")\n",
      "            \n",
      "            #Extract from Pfam lines the\n",
      "            if line_fields[ANNOT_TOOL].lower() == \"pfam\":\n",
      "                signature_desc = extract_signature_desc(line_fields[TAGS])\n",
      "                #Remove the .t1 to the gene_id\n",
      "                dot_position = line_fields[GENE_ID].rfind(\".\")\n",
      "                if dot_position != -1:\n",
      "                    line_fields[GENE_ID] = line_fields[GENE_ID][:dot_position]\n",
      "                interprot_annot[  line_fields[GENE_ID] ] = signature_desc\n",
      "        elif line.rstrip(\"\\n\") == \"##FASTA\":\n",
      "            break"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "put_prot_id = 0\n",
      "def getSignatureDescription(gene_id, signature_descriptions,parent=False):\n",
      "    global put_prot_id\n",
      "    return signature_descriptions[gene_id] if gene_id in signature_descriptions else \"putative_protein\"+str(put_prot_id)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Substitutes the orignal ID, or parent ID for the signature_desc from interprot pfam if available\n",
      "def processAugLine(line, signature_descriptions):\n",
      "    global put_prot_id\n",
      "    new_line = line\n",
      "    if \"\\tgene\" not in line:\n",
      "        id_match = re.search( r\"ID=(.+?)(\\..*?)?(;|$)\", line)\n",
      "        if id_match:\n",
      "            gene_id = id_match.group(1)\n",
      "            new_gene_id = getSignatureDescription(gene_id,signature_descriptions)\n",
      "            new_line = re.sub(r\"ID=(.+?)(\\..+?)(;|\\n)\", \"ID='\"+new_gene_id+r\"'\\g<3>\",line)\n",
      "        if \"\\ttranscript\" not in line:\n",
      "            parent_match = re.search( r\"Parent=(.+?)(\\..*?)?(;|$)\", line)\n",
      "            if parent_match:\n",
      "                parent_id = parent_match.group(1)\n",
      "                new_parent_id = getSignatureDescription(parent_id,signature_descriptions)\n",
      "                new_line = re.sub(r\"Parent=(.+?)(;|\\n)\", \"Parent='\"+new_parent_id+r\"'\\g<2>\",new_line)\n",
      "    else:\n",
      "        #New protein, increase counter\n",
      "        put_prot_id += 1\n",
      "    return new_line"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#putative_prot_counter = 0\n",
      "with open(\"merged.gff\",\"r\") as aug_fh , open(\"annotated_merged.gff\",\"w\") as out_fh:\n",
      "    for line in aug_fh:\n",
      "        if line[0] == \"#\":\n",
      "            out_fh.write(line)\n",
      "        else:\n",
      "            out_fh.write(processAugLine(line, interprot_annot))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    }
   ],
   "metadata": {}
  }
 ]
}
{'fraction_non_alphanumeric': 0.1991092215811317, 'fraction_numerical': 0.006275938860208523, 'mean_word_length': 1.8813065033537475, 'pattern_counts': {'":': 125, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5514007', 'n_tokens_mistral': 3500, 'n_tokens_neox': 3366, 'n_words': 568}