<commit_before>import sys, os
import numpy as np
from numpy.random import randn
import pandas as pd
from scipy import stats
import matplotlib as mpl
mpl.use('Agg') # helps on mac --> maybe not necessary on linux? check?
import matplotlib.pyplot as plt
import seaborn as sns
import re
import argparse
from functions_databases import PATH_NOG_CATEGORIES

#######################################################################
""" Read in Data; Set Plotting Style """
#######################################################################

parser = argparse.ArgumentParser(description="handles de_output files")
parser.add_argument('-i','--input',help='the countstable file.')
args = parser.parse_args()

reportDF=pd.io.parsers.read_table(args.input, header=0, index_col = False,sep='\t')
#reportDF=pd.io.parsers.read_table('Axolotl_no_rmdup_good_summarized_annotation.txt', header=0, index_col = False)
#reportDF=pd.io.parsers.read_table('rabbitfish_highq_rmdup_repaired_sumTrin_annotation.txt', header=0, index_col = False)
descriptions=open(PATH_NOG_CATEGORIES,'r')

sns.set(style="white")
sns.despine()

#######################################################################
""" Functions """
#######################################################################

def unpackListofLists(input):
    out1,out2,counts = [],[],[]
    for n in input:
        if n != '.':
            count = len(n)
        if count > 0:
            for x in n:
                out1.append(x[0])
                out2.append(x[1])
                counts.append(1/float(count))
    return out1, out2, counts

def describe(d):
    if d in nogDescriptions.keys():
        return nogDescriptions.get(d)

def categorize(f):
    if f in "DMNOTUVWYZ":
        return "CELL"
    elif f in "ABJKL":
        return "INFO"
    elif f in "CEFGHIPQ":
        return "METAB"
    elif f in "RS":
        return "POOR"
    else:
        return '.'

def getMultCogCategories(cogFunct):
    categories = []
    for c in cogFunct:
        categories.append([c, categorize(c)]) # if = '.', returns '.'
    return categories

def getColor(f,set1,set2,set3,set4):
    if f in "DMNOTUVWYZ":
        return set1.next()
    elif f in "ABJKL":
        return set2.next()
    elif f in "CEFGHIPQ":
        return set3.next()
    elif f in "RS":
        return set4.next()

def get_colors(functions, c1, c2, c3, c4):
    newPalette = []
    c1,c2,c3,c4 = iter(c1), iter(c2), iter(c3), iter(c4)
    functionList = sorted(list(functions))
    for c in functionList :
        newPalette.append(getColor(c, c1, c2, c3, c4))
    return newPalette


# PFAM
pTerms = re.compile('(PF\d*)\.\d*[\^]([^\^]*)[\^]([^\^]*)[\^]')

def getMultiplePFAMMatches(pfamEntry):
    if pfamEntry != '.':
        pfamMatches= re.findall(pTerms, pfamEntry)
        if pfamMatches is not None:
            pfam = []
            for m in pfamMatches:
                pfam= pfam + [[m[0], m[2]]] # make each entry a list
            return pfam
    else:
        return []

#######################################################################
""" COG """
#######################################################################
inCats = [ x.strip() for x in descriptions]
nogDescriptions = {line[1]:line[4:] for line in inCats if(line[:1]=='[')}
nogDescriptions['.'] = '.'
descriptions.close()



reportDF['eggNOG_function'].replace({'.':np.nan}).dropna()
cogs = list(reportDF['eggNOG_function'].map(getMultCogCategories))
cogFunctions, cogCategories, cogCounts = unpackListofLists(cogs)

cogDF = pd.DataFrame(cogFunctions, cogCategories, columns=(['Category']))
cogDF["Counts"]= cogCounts
cogDF.reset_index(level=0, inplace=True)
cogDF.rename(columns={'index': 'Description'}, inplace=True)
cogDF = cogDF.replace({'.':np.nan}).dropna()

## get Colors for Plotting COG entries
blues = sns.color_palette("Blues", 10)
greens = sns.color_palette("Greens", 8)
reds = sns.color_palette("Reds", 5)
greys = sns.color_palette("Greys", 2)
colors = get_colors(cogDF["Category"].unique(), blues, reds, greens, greys)

groupedCogDF = cogDF.groupby(["Category", "Description"], as_index=False)["Counts"].sum()
groupedCogDF['longDescription'] = groupedCogDF["Category"].map(describe)
sumCogPlot = sns.factorplot("Description", "Counts", hue="longDescription", size=6,aspect=2, data=groupedCogDF, kind='bar', palette=colors)
sumCogPlot.savefig('cogMultiple.png', format='png')
#sumCogPlot.savefig('rabbitfish_cogMultiple.png', format='png')


######### WHAT WAS I TRYING TO ACCOMPLISH HERE????? ###########
#observations = pd.DataFrame(cogDF["Category"].value_counts()).sort_index()<commit_msg>temp fix pipePlot<commit_after>import sys, os
import numpy as np
from numpy.random import randn
import pandas as pd
from scipy import stats
import matplotlib as mpl
mpl.use('Agg') # helps on mac --> maybe not necessary on linux? check?
import matplotlib.pyplot as plt
import seaborn as sns
import re
import argparse
from functions_databases import PATH_NOG_CATEGORIES

#######################################################################
""" Read in Data; Set Plotting Style """
#######################################################################

parser = argparse.ArgumentParser(description="handles de_output files")
parser.add_argument('-i','--input',help='the countstable file.')
args = parser.parse_args()

reportDF=pd.io.parsers.read_table(args.input, header=0, index_col = False,sep='\t')
#reportDF=pd.io.parsers.read_table('Axolotl_no_rmdup_good_summarized_annotation.txt', header=0, index_col = False)
#reportDF=pd.io.parsers.read_table('rabbitfish_highq_rmdup_repaired_sumTrin_annotation.txt', header=0, index_col = False)
descriptions=open(PATH_NOG_CATEGORIES,'r')

sns.set(style="white")
sns.despine()

#######################################################################
""" Functions """
#######################################################################

def unpackListofLists(input):
    out1,out2,counts = [],[],[]
    for n in input:
        if n != '.':
            count = len(n)
        if count > 0:
            for x in n:
                out1.append(x[0])
                out2.append(x[1])
                counts.append(1/float(count))
    return out1, out2, counts

def describe(d):
    if d in nogDescriptions.keys():
        return nogDescriptions.get(d)

def categorize(f):
    if f in "DMNOTUVWYZ":
        return "CELL"
    elif f in "ABJKL":
        return "INFO"
    elif f in "CEFGHIPQ":
        return "METAB"
    elif f in "RS":
        return "POOR"
    else:
        return '.'

def getMultCogCategories(cogFunct):
    categories = []
    for c in cogFunct:
        categories.append([c, categorize(c)]) # if = '.', returns '.'
    return categories

def getColor(f,set1,set2,set3,set4):
    if f in "DMNOTUVWYZ":
        return set1.next()
    elif f in "ABJKL":
        return set2.next()
    elif f in "CEFGHIPQ":
        return set3.next()
    elif f in "RS":
        return set4.next()

def get_colors(functions, c1, c2, c3, c4):
    newPalette = []
    c1,c2,c3,c4 = iter(c1), iter(c2), iter(c3), iter(c4)
    functionList = sorted(list(functions))
    for c in functionList :
        newPalette.append(getColor(c, c1, c2, c3, c4))
    return newPalette


# PFAM
pTerms = re.compile('(PF\d*)\.\d*[\^]([^\^]*)[\^]([^\^]*)[\^]')

def getMultiplePFAMMatches(pfamEntry):
    if pfamEntry != '.':
        pfamMatches= re.findall(pTerms, pfamEntry)
        if pfamMatches is not None:
            pfam = []
            for m in pfamMatches:
                pfam= pfam + [[m[0], m[2]]] # make each entry a list
            return pfam
    else:
        return []

#######################################################################
""" COG """
#######################################################################
inCats = [ x.strip() for x in descriptions]
nogDescriptions = {line[1]:line[4:] for line in inCats if(line[:1]=='[')}
nogDescriptions['.'] = '.'
descriptions.close()



reportDF['eggNOG_function'].replace({'.':np.nan}).dropna()
cogs = list(reportDF['eggNOG_function'].map(getMultCogCategories))
cogFunctions, cogCategories, cogCounts = unpackListofLists(cogs)

cogDF = pd.DataFrame(cogFunctions, cogCategories, columns=(['Category']))
cogDF["Counts"]= cogCounts
cogDF.reset_index(level=0, inplace=True)
cogDF.rename(columns={'index': 'Description'}, inplace=True)
cogDF = cogDF.replace({'.':np.nan}).dropna()

## get Colors for Plotting COG entries
blues = sns.color_palette("Blues", 10)
greens = sns.color_palette("Greens", 8)
reds = sns.color_palette("Reds", 5)
greys = sns.color_palette("Greys", 2)
colors = get_colors(cogDF["Category"].unique(), blues, reds, greens, greys)

#groupedCogDF = cogDF.groupby(["Category", "Description"], as_index=False)["Counts"].sum()
#groupedCogDF['longDescription'] = groupedCogDF["Category"].map(describe)
#sumCogPlot = sns.factorplot("Description", "Counts", hue="longDescription", size=6,aspect=2, data=groupedCogDF, kind='bar', palette=colors)
#sumCogPlot.savefig('cogMultiple.png', format='png')
#sumCogPlot.savefig('rabbitfish_cogMultiple.png', format='png')


######### WHAT WAS I TRYING TO ACCOMPLISH HERE????? ###########
#observations = pd.DataFrame(cogDF["Category"].value_counts()).sort_index()
{'fraction_non_alphanumeric': 0.23552374756018218, 'fraction_numerical': 0.010843634786380394, 'mean_word_length': 4.39672322995904, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4898817', 'n_tokens_mistral': 3211, 'n_tokens_neox': 3083, 'n_words': 651}