<commit_before>{
 "metadata": {
  "name": "",
  "signature": "sha256:3ef454fed1f1aa51c73d1b121390147249967f3dc9af42f3ad367f44dea51289"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "This notebook illustrates how to port data from a *query* data set to a *subject* data set, so it can be easily included as baseline data in future analysis pipelines."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from os.path import join, sep, split\n",
      "from glob import glob"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## short_read_tax_dir should be the directory where you've downloaded (or cloned) the \n",
      "## short-read-tax-assignment repository. \n",
      "short_read_tax_dir = \"/Users/caporaso/Dropbox/code/short-read-tax-assignment/\"\n",
      "\n",
      "subject_results_dir = join(short_read_tax_dir, \"data\", \"subject-results\", \"simulated-community\")\n",
      "\n",
      "## query_results_dirs should contain the directory or directories where the results \n",
      "## of the method(s) to be compared to the pre-computed results can be found. \n",
      "query_results_dirs = \\\n",
      " [\"/Users/caporaso/analysis/2014.11.21-tax-assignment-simulated\"]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for query_results_dir in query_results_dirs:\n",
      "    method_dirs = glob(join(query_results_dir, '*', '*', '*', '*'))\n",
      "    for method_dir in method_dirs:\n",
      "        fields = method_dir.split(sep)\n",
      "        dataset_id, database_id, method_id  = fields[-3], fields[-2], fields[-1] \n",
      "\n",
      "        if method_id in ('blast', 'rdp'):\n",
      "            new_location = join(subject_results_dir, dataset_id, database_id)\n",
      "            cmd = \"mv %s %s\" % (method_dir, new_location)\n",
      "            !$cmd\n",
      "            "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],<commit_msg>MAINT: changes that were intended to be in the last commit<commit_after>{
 "metadata": {
  "name": "",
  "signature": "sha256:3d7ececd5f49a5d4c532a21e718db327f51730e0a9e5dc4639592f531d12d2bc"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "This notebook illustrates how to port data from a *query* data set to a *subject* data set, so it can be easily included as baseline data in future analysis pipelines."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from os.path import join, sep, split\n",
      "from glob import glob"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## short_read_tax_dir should be the directory where you've downloaded (or cloned) the \n",
      "## short-read-tax-assignment repository. \n",
      "short_read_tax_dir = \"/Users/caporaso/Dropbox/code/short-read-tax-assignment/\"\n",
      "\n",
      "subject_results_dir = join(short_read_tax_dir, \"data\", \"subject-results\", \"simulated-community\")\n",
      "\n",
      "## query_results_dirs should contain the directory or directories where the results \n",
      "## of the method(s) to be compared to the pre-computed results can be found. \n",
      "query_results_dirs = \\\n",
      " [\"/Users/caporaso/analysis/2014.11.21-tax-assignment-simulated\"]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for query_results_dir in query_results_dirs:\n",
      "    method_dirs = glob(join(query_results_dir, '*', '*', '*', '*', '*'))\n",
      "    for method_dir in method_dirs:\n",
      "        fields = method_dir.split(sep)\n",
      "        dataset_id, database_id, method_id  = fields[-3], fields[-2], fields[-1] \n",
      "\n",
      "        if method_id in ('blast', 'rdp'):\n",
      "            new_location = join(subject_results_dir, dataset_id, database_id)\n",
      "            cmd = \"mv %s %s\" % (method_dir, new_location)\n",
      "            !$cmd\n",
      "            "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
{'fraction_non_alphanumeric': 0.17472793228536881, 'fraction_numerical': 0.0237807335751713, 'mean_word_length': 2.3900273224043715, 'pattern_counts': {'":': 68, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3511528', 'n_tokens_mistral': 1777, 'n_tokens_neox': 1670, 'n_words': 374}