<commit_before>#!/usr/bin/env python
'''
Created on Nov 27, 2012

@author: <NAME>
@author <NAME>

Step 1 of the condensing process.
'''
import argparse
import operator

def split_phylogeny(p, level='s'):
    level = level+'__'
    result = p.split(level)
    return result[0]+level+result[1].split(';')[0]


def prune_taxonomy(taxF, level):
    """
    :@type taxF: file
    :@param taxF: The taxonomy output file to parse
    :@type level: string
    :@param level: The level of the phylogenetic assignment at which to cut off
                   every assigned taxonomic string.
    
    :@rtype: dict
    :@return: A dictionary of taxonomy strings keyed on OTU ID 
    """
    uniqueTax = {}
    nuTax = {}  #non-unique taxonomies
    
    for line in taxF:
        otuID, tax, floatVal, otuIDr = line.split('\t')
        tax = split_phylogeny(tax, level)
        if not tax in uniqueTax:
            uniqueTax[tax] = otuID, floatVal, otuIDr
            nuTax[uniqueTax[tax][0]] = []
        else:
            nuTax[uniqueTax[tax][0]].append(otuID)
    
    ut = {otuID: [tax, floatVal, otuIDr] for tax,(otuID, floatVal, otuIDr) in 
          uniqueTax.iteritems()}
    
    return ut, nuTax
    <commit_msg>Removes extra newlines from condensed assign taxa file<commit_after>#!/usr/bin/env python
'''
Created on Nov 27, 2012

@author: <NAME>
@author <NAME>

Step 1 of the condensing process.
'''
import argparse
import operator

def split_phylogeny(p, level='s'):
    level = level+'__'
    result = p.split(level)
    return result[0]+level+result[1].split(';')[0]


def prune_taxonomy(taxF, level):
    """
    :@type taxF: file
    :@param taxF: The taxonomy output file to parse
    :@type level: string
    :@param level: The level of the phylogenetic assignment at which to cut off
                   every assigned taxonomic string.
    
    :@rtype: dict
    :@return: A dictionary of taxonomy strings keyed on OTU ID 
    """
    uniqueTax = {}
    nuTax = {}  #non-unique taxonomies
    
    for line in taxF:
        otuID, tax, floatVal, otuIDr = line.strip().split('\t')
        tax = split_phylogeny(tax, level)
        if not tax in uniqueTax:
            uniqueTax[tax] = otuID, floatVal, otuIDr
            nuTax[uniqueTax[tax][0]] = []
        else:
            nuTax[uniqueTax[tax][0]].append(otuID)
    
    ut = {otuID: [tax, floatVal, otuIDr] for tax,(otuID, floatVal, otuIDr) in 
          uniqueTax.iteritems()}
    
    return ut, nuTax
    
{'fraction_non_alphanumeric': 0.11948157148643175, 'fraction_numerical': 0.009720534629404616, 'mean_word_length': 2.5953420669577874, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '297682', 'n_tokens_mistral': 860, 'n_tokens_neox': 825, 'n_words': 243}