<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading required package: Rcpp\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1] ‘1.1.8’"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading required package: BiocGenerics\n",
      "Loading required package: parallel\n",
      "\n",
      "Attaching package: ‘BiocGenerics’\n",
      "\n",
      "The following objects are masked from ‘package:parallel’:\n",
      "\n",
      "    clusterApply, clusterApplyLB, clusterCall, clusterEvalQ,\n",
      "    clusterExport, clusterMap, parApply, parCapply, parLapply,\n",
      "    parLapplyLB, parRapply, parSapply, parSapplyLB\n",
      "\n",
      "The following objects are masked from ‘package:stats’:\n",
      "\n",
      "    IQR, mad, xtabs\n",
      "\n",
      "The following objects are masked from ‘package:base’:\n",
      "\n",
      "    anyDuplicated, append, as.data.frame, cbind, colnames, do.call,\n",
      "    duplicated, eval, evalq, Filter, Find, get, grep, grepl, intersect,\n",
      "    is.unsorted, lapply, lengths, Map, mapply, match, mget, order,\n",
      "    paste, pmax, pmax.int, pmin, pmin.int, Position, rank, rbind,\n",
      "    Reduce, rownames, sapply, setdiff, sort, table, tapply, union,\n",
      "    unique, unsplit, which, which.max, which.min\n",
      "\n",
      "Loading required package: BiocParallel\n",
      "Loading required package: Biostrings\n",
      "Loading required package: S4Vectors\n",
      "Loading required package: stats4\n",
      "\n",
      "Attaching package: ‘S4Vectors’\n",
      "\n",
      "The following objects are masked from ‘package:base’:\n",
      "\n",
      "    colMeans, colSums, expand.grid, rowMeans, rowSums\n",
      "\n",
      "Loading required package: IRanges\n",
      "Loading required package: XVector\n",
      "Loading required package: Rsamtools\n",
      "Loading required package: GenomeInfoDb\n",
      "Loading required package: GenomicRanges\n",
      "Loading required package: GenomicAlignments\n",
      "Loading required package: SummarizedExperiment\n",
      "Loading required package: Biobase\n",
      "Welcome to Bioconductor\n",
      "\n",
      "    Vignettes contain introductory material; view with\n",
      "    'browseVignettes()'. To cite Bioconductor, see\n",
      "    'citation(\"Biobase\")', and for packages 'citation(\"pkgname\")'.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1] ‘1.32.0’"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[1] ‘2.1.0’"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "library(dada2); packageVersion(\"dada2\")\n",
    "library(ShortRead); packageVersion(\"ShortRead\")\n",
    "library(ggplot2); packageVersion(\"ggplot2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "p1 <- \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data\"\n",
    "p2 <- \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_2_[Synmetcoitagspl2_FD]/Raw_Data\"\n",
    "fns <- c(list.files(p1, full.names=T), list.files(p2, full.names=T))\n",
    "head(fns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ERROR",
     "evalue": "Error in as.vector(x, \"character\"): cannot coerce type 'closure' to vector of type 'character'\n",
     "output_type": "error",
     "traceback": [
      "Error in as.vector(x, \"character\"): cannot coerce type 'closure' to vector of type 'character'\nTraceback:\n",
      "1. file.path(path, fnFs)",
      "2. as.character(structure(function (object, ...) \n . standardGeneric(\"path\"), generic = structure(\"path\", package = \"Rsamtools\"), package = \"Rsamtools\", group = list(), valueClass = character(0), signature = \"object\", default = `\\001NULL\\001`, skeleton = (function (object, \n .     ...) \n . stop(\"invalid call in method dispatch to 'path' (no default method)\", \n .     domain = NA))(object, ...), class = structure(\"standardGeneric\", package = \"methods\")))",
      "3. as.character.default(structure(function (object, ...) \n . standardGeneric(\"path\"), generic = structure(\"path\", package = \"Rsamtools\"), package = \"Rsamtools\", group = list(), valueClass = character(0), signature = \"object\", default = `\\001NULL\\001`, skeleton = (function (object, \n .     ...) \n . stop(\"invalid call in method dispatch to 'path' (no default method)\", \n .     domain = NA))(object, ...), class = structure(\"standardGeneric\", package = \"methods\")))"
     ]
    }
   ],
   "source": [
    "fastqs <- fns[grepl(\".fastq.gz\", fns)]\n",
    "fastqs <- sort(fastqs) # Sort ensures forward/reverse reads are in same order\n",
    "fnFs <- fastqs[grepl(\"forward\", fastqs)] # Just the forward read files\n",
    "fnRs <- fastqs[grepl(\"reverse\", fastqs)] # Just the reverse read files\n",
    "# Get sample names from the first part of the forward read filenames\n",
    "sample.names <- sapply(strsplit(fnFs, \"_\"), `[`, 1)\n",
    "# Fully specify the path for the fnFs and fnRs\n",
    "fnFs <- file.path(path, fnFs)\n",
    "fnRs <- file.path(path, fnRs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "strsplit(fnFs, \".\", fixed=T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.3.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
<commit_msg>checkpoint commit of work in progressI'm going to rip apart all of the path processing stuff next<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Illumina 16S amplicon analysis with DADA2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading required package: Rcpp\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1] ‘1.1.8’"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading required package: BiocGenerics\n",
      "Loading required package: parallel\n",
      "\n",
      "Attaching package: ‘BiocGenerics’\n",
      "\n",
      "The following objects are masked from ‘package:parallel’:\n",
      "\n",
      "    clusterApply, clusterApplyLB, clusterCall, clusterEvalQ,\n",
      "    clusterExport, clusterMap, parApply, parCapply, parLapply,\n",
      "    parLapplyLB, parRapply, parSapply, parSapplyLB\n",
      "\n",
      "The following objects are masked from ‘package:stats’:\n",
      "\n",
      "    IQR, mad, xtabs\n",
      "\n",
      "The following objects are masked from ‘package:base’:\n",
      "\n",
      "    anyDuplicated, append, as.data.frame, cbind, colnames, do.call,\n",
      "    duplicated, eval, evalq, Filter, Find, get, grep, grepl, intersect,\n",
      "    is.unsorted, lapply, lengths, Map, mapply, match, mget, order,\n",
      "    paste, pmax, pmax.int, pmin, pmin.int, Position, rank, rbind,\n",
      "    Reduce, rownames, sapply, setdiff, sort, table, tapply, union,\n",
      "    unique, unsplit, which, which.max, which.min\n",
      "\n",
      "Loading required package: BiocParallel\n",
      "Loading required package: Biostrings\n",
      "Loading required package: S4Vectors\n",
      "Loading required package: stats4\n",
      "\n",
      "Attaching package: ‘S4Vectors’\n",
      "\n",
      "The following objects are masked from ‘package:base’:\n",
      "\n",
      "    colMeans, colSums, expand.grid, rowMeans, rowSums\n",
      "\n",
      "Loading required package: IRanges\n",
      "Loading required package: XVector\n",
      "Loading required package: Rsamtools\n",
      "Loading required package: GenomeInfoDb\n",
      "Loading required package: GenomicRanges\n",
      "Loading required package: GenomicAlignments\n",
      "Loading required package: SummarizedExperiment\n",
      "Loading required package: Biobase\n",
      "Welcome to Bioconductor\n",
      "\n",
      "    Vignettes contain introductory material; view with\n",
      "    'browseVignettes()'. To cite Bioconductor, see\n",
      "    'citation(\"Biobase\")', and for packages 'citation(\"pkgname\")'.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1] ‘1.32.0’"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[1] ‘2.1.0’"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "library(dada2); packageVersion(\"dada2\")\n",
    "library(ShortRead); packageVersion(\"ShortRead\")\n",
    "library(ggplot2); packageVersion(\"ggplot2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Input file handling\n",
    "\n",
    "### Create a list of files in the itag directories (there are two in this case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.forward.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.orphaned.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.reverse.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.forward.fastq.gz'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AACAGGTTCGC.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AACAGGTTCGC.forward.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AACAGGTTCGC.orphaned.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AACAGGTTCGC.reverse.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AACTGTGCGTA.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AACTGTGCGTA.forward.fastq.gz'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.fastq.gz'\n",
       "2. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.forward.fastq.gz'\n",
       "3. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.orphaned.fastq.gz'\n",
       "4. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.reverse.fastq.gz'\n",
       "5. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.fastq.gz'\n",
       "6. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.forward.fastq.gz'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.fastq.gz\"         \n",
       "[2] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.forward.fastq.gz\" \n",
       "[3] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.orphaned.fastq.gz\"\n",
       "[4] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.reverse.fastq.gz\" \n",
       "[5] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.fastq.gz\"         \n",
       "[6] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.forward.fastq.gz\" "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p1 <- \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data\"\n",
    "p2 <- \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_2_[Synmetcoitagspl2_FD]/Raw_Data\"\n",
    "fns <- c(list.files(p1, full.names=T), list.files(p2, full.names=T))\n",
    "head(fns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create matched lists of the forward and reverse file names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.forward.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.forward.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AAGTGGACTCT.forward.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACACGGTGTCT.forward.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACATCTTGACG.forward.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACGACGTCTTA.forward.fastq.gz'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AACAGGTTCGC.forward.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AACTGTGCGTA.forward.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AAGTGGACTCT.forward.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.ACACGGTGTCT.forward.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.ACATCTTGACG.forward.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.ACGACGTCTTA.forward.fastq.gz'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.forward.fastq.gz'\n",
       "2. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.forward.fastq.gz'\n",
       "3. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AAGTGGACTCT.forward.fastq.gz'\n",
       "4. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACACGGTGTCT.forward.fastq.gz'\n",
       "5. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACATCTTGACG.forward.fastq.gz'\n",
       "6. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACGACGTCTTA.forward.fastq.gz'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.forward.fastq.gz\"\n",
       "[2] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.forward.fastq.gz\"\n",
       "[3] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AAGTGGACTCT.forward.fastq.gz\"\n",
       "[4] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACACGGTGTCT.forward.fastq.gz\"\n",
       "[5] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACATCTTGACG.forward.fastq.gz\"\n",
       "[6] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACGACGTCTTA.forward.fastq.gz\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.reverse.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.reverse.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AAGTGGACTCT.reverse.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACACGGTGTCT.reverse.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACATCTTGACG.reverse.fastq.gz'</li>\n",
       "\t<li>'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACGACGTCTTA.reverse.fastq.gz'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AACAGGTTCGC.reverse.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AACTGTGCGTA.reverse.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.AAGTGGACTCT.reverse.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.ACACGGTGTCT.reverse.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.ACATCTTGACG.reverse.fastq.gz'\n",
       "\\item 'data/Understanding\\_methane\\_cycling\\_through\\_manipulation\\_of\\_synthetic\\_methane-oxidizing\\_communities\\_\\_Synthetic\\_methane-oxidizing\\_com\\_itags\\_pl\\_1\\_{[}Synmetcoitagspl1\\_FD{]}/Raw\\_Data/10759.1.176686.ACGACGTCTTA.reverse.fastq.gz'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.reverse.fastq.gz'\n",
       "2. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.reverse.fastq.gz'\n",
       "3. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AAGTGGACTCT.reverse.fastq.gz'\n",
       "4. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACACGGTGTCT.reverse.fastq.gz'\n",
       "5. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACATCTTGACG.reverse.fastq.gz'\n",
       "6. 'data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACGACGTCTTA.reverse.fastq.gz'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACAGGTTCGC.reverse.fastq.gz\"\n",
       "[2] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AACTGTGCGTA.reverse.fastq.gz\"\n",
       "[3] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.AAGTGGACTCT.reverse.fastq.gz\"\n",
       "[4] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACACGGTGTCT.reverse.fastq.gz\"\n",
       "[5] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACATCTTGACG.reverse.fastq.gz\"\n",
       "[6] \"data/Understanding_methane_cycling_through_manipulation_of_synthetic_methane-oxidizing_communities__Synthetic_methane-oxidizing_com_itags_pl_1_[Synmetcoitagspl1_FD]/Raw_Data/10759.1.176686.ACGACGTCTTA.reverse.fastq.gz\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fastqs <- fns[grepl(\".fastq.gz\", fns)]\n",
    "fastqs <- sort(fastqs) # Sort ensures forward/reverse reads are in same order\n",
    "fnFs <- fastqs[grepl(\"forward\", fastqs)] # Just the forward read files\n",
    "fnRs <- fastqs[grepl(\"reverse\", fastqs)] # Just the reverse read files\n",
    "head(fnFs)\n",
    "head(fnRs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read in the sample to filename map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sample.map <- read.table(\"1.create_sample_map.txt\", header=F, sep='\\t', col.names=c(\"sample\", \"filename\", \"forward\", \"reverse\"),  colClasses = \"character\")\n",
    "sample.map$barcode <- sapply(strsplit(sample.map$filename, \".\", fixed=T), `[`, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>sample</th><th scope=col>filename</th><th scope=col>forward</th><th scope=col>reverse</th><th scope=col>barcode</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>1-HH0420b                                  </td><td>10759.1.176686.TAAGGCCTATC.fastq.gz        </td><td>10759.1.176686.TAAGGCCTATC.forward.fastq.gz</td><td>10759.1.176686.TAAGGCCTATC.reverse.fastq.gz</td><td>TAAGGCCTATC                                </td></tr>\n",
       "\t<tr><td>2-HH0424b                                  </td><td>10759.1.176686.TTGTCGCACAA.fastq.gz        </td><td>10759.1.176686.TTGTCGCACAA.forward.fastq.gz</td><td>10759.1.176686.TTGTCGCACAA.reverse.fastq.gz</td><td>TTGTCGCACAA                                </td></tr>\n",
       "\t<tr><td>3-HH0427b                                  </td><td>10759.1.176686.AACTGTGCGTA.fastq.gz        </td><td>10759.1.176686.AACTGTGCGTA.forward.fastq.gz</td><td>10759.1.176686.AACTGTGCGTA.reverse.fastq.gz</td><td>AACTGTGCGTA                                </td></tr>\n",
       "\t<tr><td>4-HH0429b                                  </td><td>10759.1.176686.TGCGAACGTAT.fastq.gz        </td><td>10759.1.176686.TGCGAACGTAT.forward.fastq.gz</td><td>10759.1.176686.TGCGAACGTAT.reverse.fastq.gz</td><td>TGCGAACGTAT                                </td></tr>\n",
       "\t<tr><td>5-HH0501b                                  </td><td>10759.1.176686.CATCAGCGTGT.fastq.gz        </td><td>10759.1.176686.CATCAGCGTGT.forward.fastq.gz</td><td>10759.1.176686.CATCAGCGTGT.reverse.fastq.gz</td><td>CATCAGCGTGT                                </td></tr>\n",
       "\t<tr><td>6-HH0503b                                  </td><td>10759.1.176686.TTCAACAGGTG.fastq.gz        </td><td>10759.1.176686.TTCAACAGGTG.forward.fastq.gz</td><td>10759.1.176686.TTCAACAGGTG.reverse.fastq.gz</td><td>TTCAACAGGTG                                </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllll}\n",
       " sample & filename & forward & reverse & barcode\\\\\n",
       "\\hline\n",
       "\t 1-HH0420b                                   & 10759.1.176686.TAAGGCCTATC.fastq.gz         & 10759.1.176686.TAAGGCCTATC.forward.fastq.gz & 10759.1.176686.TAAGGCCTATC.reverse.fastq.gz & TAAGGCCTATC                                \\\\\n",
       "\t 2-HH0424b                                   & 10759.1.176686.TTGTCGCACAA.fastq.gz         & 10759.1.176686.TTGTCGCACAA.forward.fastq.gz & 10759.1.176686.TTGTCGCACAA.reverse.fastq.gz & TTGTCGCACAA                                \\\\\n",
       "\t 3-HH0427b                                   & 10759.1.176686.AACTGTGCGTA.fastq.gz         & 10759.1.176686.AACTGTGCGTA.forward.fastq.gz & 10759.1.176686.AACTGTGCGTA.reverse.fastq.gz & AACTGTGCGTA                                \\\\\n",
       "\t 4-HH0429b                                   & 10759.1.176686.TGCGAACGTAT.fastq.gz         & 10759.1.176686.TGCGAACGTAT.forward.fastq.gz & 10759.1.176686.TGCGAACGTAT.reverse.fastq.gz & TGCGAACGTAT                                \\\\\n",
       "\t 5-HH0501b                                   & 10759.1.176686.CATCAGCGTGT.fastq.gz         & 10759.1.176686.CATCAGCGTGT.forward.fastq.gz & 10759.1.176686.CATCAGCGTGT.reverse.fastq.gz & CATCAGCGTGT                                \\\\\n",
       "\t 6-HH0503b                                   & 10759.1.176686.TTCAACAGGTG.fastq.gz         & 10759.1.176686.TTCAACAGGTG.forward.fastq.gz & 10759.1.176686.TTCAACAGGTG.reverse.fastq.gz & TTCAACAGGTG                                \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "  sample    filename                           \n",
       "1 1-HH0420b 10759.1.176686.TAAGGCCTATC.fastq.gz\n",
       "2 2-HH0424b 10759.1.176686.TTGTCGCACAA.fastq.gz\n",
       "3 3-HH0427b 10759.1.176686.AACTGTGCGTA.fastq.gz\n",
       "4 4-HH0429b 10759.1.176686.TGCGAACGTAT.fastq.gz\n",
       "5 5-HH0501b 10759.1.176686.CATCAGCGTGT.fastq.gz\n",
       "6 6-HH0503b 10759.1.176686.TTCAACAGGTG.fastq.gz\n",
       "  forward                                    \n",
       "1 10759.1.176686.TAAGGCCTATC.forward.fastq.gz\n",
       "2 10759.1.176686.TTGTCGCACAA.forward.fastq.gz\n",
       "3 10759.1.176686.AACTGTGCGTA.forward.fastq.gz\n",
       "4 10759.1.176686.TGCGAACGTAT.forward.fastq.gz\n",
       "5 10759.1.176686.CATCAGCGTGT.forward.fastq.gz\n",
       "6 10759.1.176686.TTCAACAGGTG.forward.fastq.gz\n",
       "  reverse                                     barcode    \n",
       "1 10759.1.176686.TAAGGCCTATC.reverse.fastq.gz TAAGGCCTATC\n",
       "2 10759.1.176686.TTGTCGCACAA.reverse.fastq.gz TTGTCGCACAA\n",
       "3 10759.1.176686.AACTGTGCGTA.reverse.fastq.gz AACTGTGCGTA\n",
       "4 10759.1.176686.TGCGAACGTAT.reverse.fastq.gz TGCGAACGTAT\n",
       "5 10759.1.176686.CATCAGCGTGT.reverse.fastq.gz CATCAGCGTGT\n",
       "6 10759.1.176686.TTCAACAGGTG.reverse.fastq.gz TTCAACAGGTG"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "head(sample.map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for(val in 1:length(fnFs)) {\n",
    "    plotQualityProfile(fnFs[[1]])\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reverse reads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for(val in 1:length(fnRs)) {\n",
    "    plotQualityProfile(fnRs[[1]])\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform filtering and trimming\n",
    "\n",
    "We’ll use standard filtering parameters: maxN=0 (DADA2 requires no Ns), truncQ=2 (quality score 2 in Illumina means “stop using this read”) and maxEE=2. The maxEE parameter sets the maximum number of “expected errors” allowed in a read, which is a better filter than simply averaging quality scores. We use the fastqPairedFilter function to jointly filter the forward and reverse reads.\n",
    "\n",
    "### Filter the forward and reverse reads:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filt_path <- file.path(path, \"filtered\")\n",
    "if(!file_test(\"-d\", filt_path)) dir.create(filt_path)\n",
    "filtFs <- file.path(filt_path, paste0(sample.map$sample, \"_F_filt.fastq.gz\"))\n",
    "filtRs <- file.path(filt_path, paste0(sample.map, \"_R_filt.fastq.gz\"))\n",
    "# Filter\n",
    "for(i in seq_along(fnFs)) {\n",
    "  fastqPairedFilter(c(fnFs[i], fnRs[i]), c(filtFs[i], filtRs[i]),\n",
    "                    trimLeft=c(10, 10), truncLen=c(240,160), \n",
    "                    maxN=0, maxEE=2, truncQ=2, \n",
    "                    compress=TRUE, verbose=TRUE)\n",
    "}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.3.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}

{'fraction_non_alphanumeric': 0.15901797594510683, 'fraction_numerical': 0.050196978788906575, 'mean_word_length': 3.453869393446433, 'pattern_counts': {'":': 193, '<': 157, '<?xml version=': 0, '>': 135, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7165204', 'n_tokens_mistral': 16469, 'n_tokens_neox': 14576, 'n_words': 1396}