<commit_before>{
 "metadata": {
  "name": "",
  "signature": "sha256:8188f0184b3da8f776a5a1e7151300acd598138a9b8fdbcb1e263aa9935cd1c9"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from collections import defaultdict\n",
      "\n",
      "from Bio.PopGen.GenePop import read\n",
      "from Bio.PopGen.GenePop.LargeFileParser import read as read_large\n",
      "\n",
      "from genomics.popgen.plink.convert import to_genepop"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "f = open('relationships_w_pops_121708.txt')\n",
      "pop_ind = defaultdict(list)\n",
      "f.readline()  # header\n",
      "for l in f:\n",
      "    toks = l.rstrip().split('\\t')\n",
      "    fam_id = toks[0]\n",
      "    ind_id = toks[1]\n",
      "    pop = toks[-1]\n",
      "    pop_ind[pop].append((fam_id, ind_id))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Check for consistency issues"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "all_inds = []\n",
      "for inds in pop_ind.values():\n",
      "    all_inds.extend(inds)\n",
      "for l in open('hapmap1.ped'):\n",
      "    toks = l.rstrip().replace(' ', '\\t').split('\\t')\n",
      "    fam = toks[0]\n",
      "    ind = toks[1]\n",
      "    if (fam, ind) not in all_inds:\n",
      "        print('Problems with %s/%s' % (fam, ind))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Problems with 2469/NA20281\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Convert from PLINK to Genepop"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "to_genepop('hapmap1_auto', 'hapmap1_auto', pop_ind)\n",
      "to_genepop('hapmap10', 'hapmap10', pop_ind)\n",
      "to_genepop('hapmap10_auto', 'hapmap10_auto', pop_ind)\n",
      "to_genepop('hapmap10_auto_noofs_ld', 'hapmap10_auto_noofs_ld', pop_ind)\n",
      "#slow"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Using the in-memory parser"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Careful: this will severely increase your memory usage, consider it optional"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rec = read(open('hapmap1_auto.gp'))\n",
      "print('Number of loci %d' % len(rec.loci_list))\n",
      "print('Number of populations %d' % len(rec.pop_list))\n",
      "print('Individuals per population %s' % ', '.join([str(len(inds)) for inds in rec.populations]))\n",
      "del rec\n",
      "#talk about population names"
     ],
     "language": "python",
     "metadata": {},<commit_msg>genepop<commit_after>{
 "metadata": {
  "name": "",
  "signature": "sha256:9aa6f73af955d5f79483bac429f034d94c0b22134198aecbc41b04a6059616e6"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from collections import defaultdict\n",
      "\n",
      "from Bio.PopGen.GenePop import read\n",
      "from Bio.PopGen.GenePop.LargeFileParser import read as read_large\n",
      "\n",
      "from genomics.popgen.plink.convert import to_genepop"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "f = open('relationships_w_pops_121708.txt')\n",
      "pop_ind = defaultdict(list)\n",
      "f.readline()  # header\n",
      "for l in f:\n",
      "    toks = l.rstrip().split('\\t')\n",
      "    fam_id = toks[0]\n",
      "    ind_id = toks[1]\n",
      "    pop = toks[-1]\n",
      "    pop_ind[pop].append((fam_id, ind_id))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Check for consistency issues"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "all_inds = []\n",
      "for inds in pop_ind.values():\n",
      "    all_inds.extend(inds)\n",
      "for l in open('hapmap1.ped'):\n",
      "    toks = l.rstrip().replace(' ', '\\t').split('\\t')\n",
      "    fam = toks[0]\n",
      "    ind = toks[1]\n",
      "    if (fam, ind) not in all_inds:\n",
      "        print('Problems with %s/%s' % (fam, ind))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Problems with 2469/NA20281\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Convert from PLINK to Genepop"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "to_genepop('hapmap1_auto', 'hapmap1_auto', pop_ind)\n",
      "to_genepop('hapmap10', 'hapmap10', pop_ind)\n",
      "to_genepop('hapmap10_auto', 'hapmap10_auto', pop_ind)\n",
      "to_genepop('hapmap10_auto_noofs_ld', 'hapmap10_auto_noofs_ld', pop_ind)\n",
      "to_genepop('hapmap10_auto_noofs_2', 'hapmap10_auto_noofs_2', pop_ind)\n",
      "#slow"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Using the in-memory parser"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Careful: this will severely increase your memory usage, consider it optional"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rec = read(open('hapmap1_auto.gp'))\n",
      "print('Number of loci %d' % len(rec.loci_list))\n",
      "print('Number of populations %d' % len(rec.pop_list))\n",
      "print('Individuals per population %s' % ', '.join([str(len(inds)) for inds in rec.populations]))\n",
      "del rec\n",
      "#talk about population names"
     ],
     "language": "python",
     "metadata": {},
{'fraction_non_alphanumeric': 0.19818285065303803, 'fraction_numerical': 0.026405451448040886, 'mean_word_length': 2.1877828054298645, 'pattern_counts': {'":': 123, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2480170', 'n_tokens_mistral': 2771, 'n_tokens_neox': 2635, 'n_words': 466}