<commit_before>#!/usr/bin/env python3.4
'''
Sanitize the result csvs

usage:
    ./sanitize_csv_1_0_0.py <input_file> <output_file> <validation_score_field> <bigger_scores_better> <score_diff_threshold> <log10_threshold> <accept_conflicting_psms> <num_compared_psms> <remove_redundant_psms>


'''
import sys
import os
import csv
import ursgal
import math
import operator
from collections import defaultdict as ddict

if sys.platform != 'win32':
    csv.field_size_limit(sys.maxsize)


def main( 
    input_file=None, 
    output_file=None,
    grouped_psms=None,
    validation_score_field=None, 
    bigger_scores_better=None,
    score_diff_threshold=2.0,
    log10_threshold=True,
    accept_conflicting_psms=False,
    num_compared_psms=2,
    # remove_redundant_psms=False,
    psm_defining_colnames=[
        'Spectrum Title',
        'Sequence',
        'Modifications',
    ]
):
    '''
    Spectra with multiple PSMs are sanitized, i.e. only the PSM with best PEP score is accepted
    and only if the best hit has a PEP that is at least two orders of magnitude smaller than the others
    '''

    # un = ursgal.UNode()
    # grouped_psms = un._group_psms( input_file, validation_score_field=validation_score_field, bigger_scores_better=bigger_scores_better )

    if grouped_psms is None:
        grouped_psms = group_psms( input_file, validation_score_field=validation_score_field, bigger_scores_better=bigger_scores_better )

    all_line_dicts = []
    for spec_title, grouped_psm_list in grouped_psms.items():
        spec_line_dicts = []
        best_score = None
        psm_names = set()
        for n, grouped_psm in enumerate(grouped_psm_list):
            score, line_dict = grouped_psm
            if n == 0:
                best_score = score
                psm = ''
                for column in psm_defining_colnames:
                    psm += '|{0}'.format(line_dict[column])
                psm_names.add(psm)
                spec_line_dicts.append(line_dict)
            elif n < num_compared_psms:
                psm = line_dict['Sequence']+line_dict['Modifications']+line_dict['Charge']
                # if psm in psm_names and remove_redundant_psms is True:
                #     continue
                if log10_threshold is True:
                    if abs(math.log10(best_score)) - abs(math.log10(score)) >= score_diff_threshold:
                        break
                    else:
                        spec_line_dicts.append(line_dict)
                        psm_names.add(psm)
                else:
                    if abs(best_score) - abs(score) >= score_diff_threshold:
                        break
                    else:
                        spec_line_dicts.append(line_dict)
                        psm_names.add(psm)
        if accept_conflicting_psms is False and len(spec_line_dicts) >= 2:
            continue   
        else:
            all_line_dicts.extend(spec_line_dicts)<commit_msg>handling score 0 in sanitize_csv<commit_after>#!/usr/bin/env python3.4
'''
Sanitize the result csvs

usage:
    ./sanitize_csv_1_0_0.py <input_file> <output_file> <validation_score_field> <bigger_scores_better> <score_diff_threshold> <log10_threshold> <accept_conflicting_psms> <num_compared_psms> <remove_redundant_psms>


'''
import sys
import os
import csv
import ursgal
import math
import operator
from collections import defaultdict as ddict

if sys.platform != 'win32':
    csv.field_size_limit(sys.maxsize)


def main( 
    input_file=None, 
    output_file=None,
    grouped_psms=None,
    validation_score_field=None, 
    bigger_scores_better=None,
    score_diff_threshold=2.0,
    log10_threshold=True,
    accept_conflicting_psms=False,
    num_compared_psms=2,
    # remove_redundant_psms=False,
    psm_defining_colnames=[
        'Spectrum Title',
        'Sequence',
        'Modifications',
    ]
):
    '''
    Spectra with multiple PSMs are sanitized, i.e. only the PSM with best PEP score is accepted
    and only if the best hit has a PEP that is at least two orders of magnitude smaller than the others
    '''

    # un = ursgal.UNode()
    # grouped_psms = un._group_psms( input_file, validation_score_field=validation_score_field, bigger_scores_better=bigger_scores_better )

    if grouped_psms is None:
        grouped_psms = group_psms( input_file, validation_score_field=validation_score_field, bigger_scores_better=bigger_scores_better )

    all_line_dicts = []
    for spec_title, grouped_psm_list in grouped_psms.items():
        spec_line_dicts = []
        best_score = None
        psm_names = set()
        for n, grouped_psm in enumerate(grouped_psm_list):
            score, line_dict = grouped_psm
            if n == 0:
                best_score = score
                psm = ''
                for column in psm_defining_colnames:
                    psm += '|{0}'.format(line_dict[column])
                psm_names.add(psm)
                spec_line_dicts.append(line_dict)
            elif n < num_compared_psms:
                psm = line_dict['Sequence']+line_dict['Modifications']+line_dict['Charge']
                # if psm in psm_names and remove_redundant_psms is True:
                #     continue
                if log10_threshold is True:
                    if best_score != 0:
                        log_best_score = abs(math.log10(best_score))
                    else:
                        log_best_score = 0
                    if score != 0:
                        log_score = abs(math.log10(score))
                    else:
                        log_score = 0
                    if log_best_score - log_score >= score_diff_threshold:
                         break
                    else:
                        spec_line_dicts.append(line_dict)
                        psm_names.add(psm)
                else:
                    if abs(best_score) - abs(score) >= score_diff_threshold:
                        break
                    else:
                        spec_line_dicts.append(line_dict)
                        psm_names.add(psm)
        if accept_conflicting_psms is False and len(spec_line_dicts) >= 2:
            continue   
        else:
            all_line_dicts.extend(spec_line_dicts)
{'fraction_non_alphanumeric': 0.07527226137091608, 'fraction_numerical': 0.008167841127482383, 'mean_word_length': 2.1007944389275073, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6277420', 'n_tokens_mistral': 2020, 'n_tokens_neox': 1886, 'n_words': 393}