<commit_before>import numpy as np
from numpy import ma
import scipy as sp
import scipy.stats
import scipy.special
import matplotlib as mpl
import matplotlib.pyplot as plt

import pyutils
import vdj

def vdjxml2countdict(inhandle,features,count='read'):
    counts = pyutils.nesteddict()
    uniq_feature_values = dict([(f,set()) for f in features])
    for chain in vdj.parse_VDJXML(inhandle):
        try:    # get the feature tuple
            feature_list = [chain.__getattribute__(f) for f in features]
            for (feature,value) in zip(features,feature_list): uniq_feature_values[feature].add(value)
        except AttributeError:  # chain is missing a feature; abandon it
            continue
        
        # update the dictionary
        if count == 'read':
            counts.nested_increment(feature_list)
        elif count in ['junction','clone']:
            counts.nested_add(feature_list,chain.__getattribute__(count))
        else:
            raise ValueError, "'count' must be 'read', 'junction', or 'clone'"
        
        # if counting clones/junctions, convert sets to numbers
        if count in ['junction','clone']:
            for tup in counts.walk():
                (keylist,val) = (tup[:-1],tup[-1])
                counts.nested_assign(keylist,len(val))<commit_msg>Fixed bug: conv of sets to numbers nested wrong<commit_after>import numpy as np
from numpy import ma
import scipy as sp
import scipy.stats
import scipy.special
import matplotlib as mpl
import matplotlib.pyplot as plt

import pyutils
import vdj

def vdjxml2countdict(inhandle,features,count='read'):
    counts = pyutils.nesteddict()
    uniq_feature_values = dict([(f,set()) for f in features])
    for chain in vdj.parse_VDJXML(inhandle):
        try:    # get the feature tuple
            feature_list = [chain.__getattribute__(f) for f in features]
            for (feature,value) in zip(features,feature_list): uniq_feature_values[feature].add(value)
        except AttributeError:  # chain is missing a feature; abandon it
            continue
        
        # update the dictionary
        if count == 'read':
            counts.nested_increment(feature_list)
        elif count in ['junction','clone']:
            counts.nested_add(feature_list,chain.__getattribute__(count))
        else:
            raise ValueError, "'count' must be 'read', 'junction', or 'clone'"
        
    # if counting clones/junctions, convert sets to numbers
    if count in ['junction','clone']:
        for tup in counts.walk():
            (keylist,val) = (tup[:-1],tup[-1])
            counts.nested_assign(keylist,len(val))
{'fraction_non_alphanumeric': 0.09801678108314264, 'fraction_numerical': 0.002288329519450801, 'mean_word_length': 2.840409956076135, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1395119', 'n_tokens_mistral': 760, 'n_tokens_neox': 757, 'n_words': 223}