<commit_before>﻿using Chemistry;
using MassSpectrometry;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;

namespace EngineLayer.ModernSearch
{
    public class ModernSearchEngine : MetaMorpheusEngine
    {
        #region Protected Fields

        protected readonly List<int>[] fragmentIndex;
        protected readonly float[] keys;
        protected readonly Psm[] globalPsms;
        protected readonly Ms2ScanWithSpecificMass[] listOfSortedms2Scans;
        protected readonly List<CompactPeptide> peptideIndex;
        protected readonly List<ProductType> lp;
        protected readonly int currentPartition;
        protected readonly CommonParameters CommonParameters;
        protected readonly bool addCompIons;
        protected readonly MassDiffAcceptor massDiffAcceptors;

        #endregion Protected Fields

        #region Private Fields

        private const double tolForScoreImprovement = 1e-9;

        #endregion Private Fields

        #region Public Constructors

        public ModernSearchEngine(Psm[] globalPsms, Ms2ScanWithSpecificMass[] listOfSortedms2Scans, List<CompactPeptide> peptideIndex, float[] keys, List<int>[] fragmentIndex, List<ProductType> lp, int currentPartition, CommonParameters CommonParameters, bool addCompIons, MassDiffAcceptor massDiffAcceptors, List<string> nestedIds) : base(nestedIds)
        {
            this.globalPsms = globalPsms;
            this.listOfSortedms2Scans = listOfSortedms2Scans;
            this.peptideIndex = peptideIndex;
            this.keys = keys;
            this.fragmentIndex = fragmentIndex;
            this.lp = lp;
            this.currentPartition = currentPartition + 1;
            this.CommonParameters = CommonParameters;
            this.addCompIons = addCompIons;
            this.massDiffAcceptors = massDiffAcceptors;
        }

        #endregion Public Constructors

        #region Protected Methods

        protected override MetaMorpheusEngineResults RunSpecific()
        {
            Status("In modern search engine..." + currentPartition + "/" + CommonParameters.TotalPartitions, nestedIds);

            var listOfSortedms2ScansLength = listOfSortedms2Scans.Length;

            var outputObject = new object();
            int scansSeen = 0;
            int old_progress = 0;
            var peptideIndexCount = peptideIndex.Count;
            Parallel.ForEach(Partitioner.Create(0, listOfSortedms2ScansLength), fff =>
            {
                List<CompactPeptide> bestPeptides;
                double bestScores;
                List<int> bestNotches;
                double[] fullPeptideScores = new double[peptideIndexCount];
                for (int i = fff.Item1; i < fff.Item2; i++)
                {
                    var thisScan = listOfSortedms2Scans[i];
                    var thisScanprecursorMass = thisScan.PrecursorMass;
                    Array.Clear(fullPeptideScores, 0, peptideIndexCount);
                    double thePrecursorMass = thisScan.PrecursorMass;
                    CalculatePeptideScores(thisScan.TheScan, fullPeptideScores, thePrecursorMass);

                    bestPeptides = null;
                    bestScores = 0;
                    bestNotches = null;

                    for (int possibleWinningPeptideIndex = 0; possibleWinningPeptideIndex < fullPeptideScores.Length; possibleWinningPeptideIndex++)
                    {
                        var consideredScore = fullPeptideScores[possibleWinningPeptideIndex];
                        if (consideredScore > CommonParameters.ScoreCutoff) //intentionally high. 99.9% of 4-mers are present in a given UniProt database. This saves considerable time
                        {
                            CompactPeptide candidatePeptide = peptideIndex[possibleWinningPeptideIndex];

                            // Check if makes sense to add due to peptidescore!
                            var searchMode = massDiffAcceptors;
                            double currentBestScore = bestScores;
                            if (currentBestScore > 1)
                            {
                                // Existed! Need to compare with old match
                                if ((Math.Abs(currentBestScore - consideredScore) < tolForScoreImprovement) && (CommonParameters.ReportAllAmbiguity || bestPeptides.Count == 0))
                                {
                                    // Score is same, need to see if accepts and if prefer the new one
                                    int notch = searchMode.Accepts(thisScanprecursorMass, candidatePeptide.MonoisotopicMassIncludingFixedMods);
                                    if (notch >= 0)
                                    {
                                        bestPeptides.Add(candidatePeptide);
                                        bestNotches.Add(notch);
                                    }
                                }
                                else if (currentBestScore < consideredScore)
                                {
                                    // Score is better, only make sure it is acceptable
                                    int notch = searchMode.Accepts(thisScanprecursorMass, candidatePeptide.MonoisotopicMassIncludingFixedMods);
                                    if (notch >= 0)
                                    {
                                        bestPeptides = new List<CompactPeptide> { candidatePeptide };
                                        bestScores = consideredScore;
                                        bestNotches = new List<int> { notch };
                                    }
                                }
                            }
                            // Did not exist! Only make sure that it is acceptable
                            else
                            {
                                int notch = searchMode.Accepts(thisScanprecursorMass, candidatePeptide.MonoisotopicMassIncludingFixedMods);
                                if (notch >= 0)
                                {
                                    bestPeptides = new List<CompactPeptide> { candidatePeptide };
                                    bestScores = consideredScore;
                                    bestNotches = new List<int> { notch };
                                }
                            }
                        }
                    }
                    if (bestPeptides != null)
                    {
                        int startIndex = 0;

                        if (globalPsms[i] == null)
                        {
                            globalPsms[i] = new Psm(bestPeptides[0], bestNotches[0], bestScores, i, thisScan, CommonParameters.ExcelCompatible);
                            startIndex = 1;
                        }

                        for (int k = startIndex; k < bestPeptides.Count; k++)
                        {
                            globalPsms[i].AddOrReplace(bestPeptides[k], bestScores, bestNotches[k], CommonParameters.ReportAllAmbiguity);
                        }
                    }
                }
                lock (outputObject)
                {
                    scansSeen += fff.Item2 - fff.Item1;
                    var new_progress = (int)((double)scansSeen / (listOfSortedms2ScansLength) * 100);
                    if (new_progress > old_progress)
                    {
                        ReportProgress(new ProgressEventArgs(new_progress, "In modern search loop" + currentPartition + "/" + CommonParameters.TotalPartitions, nestedIds));
                        old_progress = new_progress;
                    }
                }
            });
            return new MetaMorpheusEngineResults(this);
        }

        protected void CalculatePeptideScores(IMsDataScan<IMzSpectrum<IMzPeak>> spectrum, double[] peptideScores, double thePrecursorMass)
        {
            //create previous variables to determine if peaks can be sequestered
            double previousTheAdd = 1 + spectrum.MassSpectrum[0].Intensity / spectrum.TotalIonCurrent;
            double previousExperimentalPeakInDaltons = spectrum.MassSpectrum[0].Mz - Constants.protonMass;
            double previousMinRange = CommonParameters.ProductMassTolerance.GetMinimumValue(previousExperimentalPeakInDaltons);
            double previousMaxRange;
            //search observed peaks
            for (int i = 1; i < spectrum.MassSpectrum.Size; i++)
            {
                double experimentalPeakInDaltons = spectrum.MassSpectrum[i].Mz - Constants.protonMass;
                if (CommonParameters.ProductMassTolerance.Within(previousExperimentalPeakInDaltons, experimentalPeakInDaltons))
                {
                    previousTheAdd += spectrum.MassSpectrum[i].Intensity / spectrum.TotalIonCurrent; //open to debate, currently sum intensities of all peaks within tolerance like it was low res
                }
                else
                {
                    previousMaxRange = CommonParameters.ProductMassTolerance.GetMaximumValue(previousExperimentalPeakInDaltons);
                    FindPeakMatches(previousTheAdd, previousMinRange, previousMaxRange, peptideScores);
                    previousTheAdd = 1 + spectrum.MassSpectrum[i].Intensity / spectrum.TotalIonCurrent;
                    previousMinRange = CommonParameters.ProductMassTolerance.GetMinimumValue(experimentalPeakInDaltons);
                }
                previousExperimentalPeakInDaltons = experimentalPeakInDaltons;
            }
            previousMaxRange = CommonParameters.ProductMassTolerance.GetMaximumValue(previousExperimentalPeakInDaltons);
            FindPeakMatches(previousTheAdd, previousMinRange, previousMaxRange, peptideScores);


            //generate experimental complementary ions if specified
            if (addCompIons)
            {
                //okay, we're not actually adding in complementary m/z peaks, we're doing a shortcut and just straight up adding the mass assuming that they're z=1
                bool HCD = lp.Contains(ProductType.B) || lp.Contains(ProductType.Y) ? true : false;
                bool ETD = lp.Contains(ProductType.C) || lp.Contains(ProductType.Zdot) ? true : false;
                IMzPeak[][] complimentaryPeaks = new IMzPeak[2][];

                if (HCD)
                {
                    complimentaryPeaks[0] = new IMzPeak[spectrum.MassSpectrum.Size];
                    for (int i = 0; i < spectrum.MassSpectrum.Size; i++)
                        complimentaryPeaks[0][i] = new MzPeak((thePrecursorMass - spectrum.MassSpectrum[i].Mz + Constants.protonMass), (spectrum.MassSpectrum[i].Intensity));
                }

                if (ETD)
                {
                    complimentaryPeaks[1] = new IMzPeak[spectrum.MassSpectrum.Size];
                    for (int i = 0; i < spectrum.MassSpectrum.Size; i++)<commit_msg>You win this round, Codacy.<commit_after>﻿using Chemistry;
using MassSpectrometry;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;

namespace EngineLayer.ModernSearch
{
    public class ModernSearchEngine : MetaMorpheusEngine
    {
        #region Protected Fields

        protected readonly List<int>[] fragmentIndex;
        protected readonly float[] keys;
        protected readonly Psm[] globalPsms;
        protected readonly Ms2ScanWithSpecificMass[] listOfSortedms2Scans;
        protected readonly List<CompactPeptide> peptideIndex;
        protected readonly List<ProductType> lp;
        protected readonly int currentPartition;
        protected readonly CommonParameters CommonParameters;
        protected readonly bool addCompIons;
        protected readonly MassDiffAcceptor massDiffAcceptors;

        #endregion Protected Fields

        #region Private Fields

        private const double tolForScoreImprovement = 1e-9;

        #endregion Private Fields

        #region Public Constructors

        public ModernSearchEngine(Psm[] globalPsms, Ms2ScanWithSpecificMass[] listOfSortedms2Scans, List<CompactPeptide> peptideIndex, float[] keys, List<int>[] fragmentIndex, List<ProductType> lp, int currentPartition, CommonParameters CommonParameters, bool addCompIons, MassDiffAcceptor massDiffAcceptors, List<string> nestedIds) : base(nestedIds)
        {
            this.globalPsms = globalPsms;
            this.listOfSortedms2Scans = listOfSortedms2Scans;
            this.peptideIndex = peptideIndex;
            this.keys = keys;
            this.fragmentIndex = fragmentIndex;
            this.lp = lp;
            this.currentPartition = currentPartition + 1;
            this.CommonParameters = CommonParameters;
            this.addCompIons = addCompIons;
            this.massDiffAcceptors = massDiffAcceptors;
        }

        #endregion Public Constructors

        #region Protected Methods

        protected override MetaMorpheusEngineResults RunSpecific()
        {
            Status("In modern search engine..." + currentPartition + "/" + CommonParameters.TotalPartitions, nestedIds);

            var listOfSortedms2ScansLength = listOfSortedms2Scans.Length;

            var outputObject = new object();
            int scansSeen = 0;
            int old_progress = 0;
            var peptideIndexCount = peptideIndex.Count;
            Parallel.ForEach(Partitioner.Create(0, listOfSortedms2ScansLength), fff =>
            {
                List<CompactPeptide> bestPeptides;
                double bestScores;
                List<int> bestNotches;
                double[] fullPeptideScores = new double[peptideIndexCount];
                for (int i = fff.Item1; i < fff.Item2; i++)
                {
                    var thisScan = listOfSortedms2Scans[i];
                    var thisScanprecursorMass = thisScan.PrecursorMass;
                    Array.Clear(fullPeptideScores, 0, peptideIndexCount);
                    double thePrecursorMass = thisScan.PrecursorMass;
                    CalculatePeptideScores(thisScan.TheScan, fullPeptideScores, thePrecursorMass);

                    bestPeptides = null;
                    bestScores = 0;
                    bestNotches = null;

                    for (int possibleWinningPeptideIndex = 0; possibleWinningPeptideIndex < fullPeptideScores.Length; possibleWinningPeptideIndex++)
                    {
                        var consideredScore = fullPeptideScores[possibleWinningPeptideIndex];
                        if (consideredScore > CommonParameters.ScoreCutoff) //intentionally high. 99.9% of 4-mers are present in a given UniProt database. This saves considerable time
                        {
                            CompactPeptide candidatePeptide = peptideIndex[possibleWinningPeptideIndex];

                            // Check if makes sense to add due to peptidescore!
                            var searchMode = massDiffAcceptors;
                            double currentBestScore = bestScores;
                            if (currentBestScore > 1)
                            {
                                // Existed! Need to compare with old match
                                if ((Math.Abs(currentBestScore - consideredScore) < tolForScoreImprovement) && (CommonParameters.ReportAllAmbiguity || bestPeptides.Count == 0))
                                {
                                    // Score is same, need to see if accepts and if prefer the new one
                                    int notch = searchMode.Accepts(thisScanprecursorMass, candidatePeptide.MonoisotopicMassIncludingFixedMods);
                                    if (notch >= 0)
                                    {
                                        bestPeptides.Add(candidatePeptide);
                                        bestNotches.Add(notch);
                                    }
                                }
                                else if (currentBestScore < consideredScore)
                                {
                                    // Score is better, only make sure it is acceptable
                                    int notch = searchMode.Accepts(thisScanprecursorMass, candidatePeptide.MonoisotopicMassIncludingFixedMods);
                                    if (notch >= 0)
                                    {
                                        bestPeptides = new List<CompactPeptide> { candidatePeptide };
                                        bestScores = consideredScore;
                                        bestNotches = new List<int> { notch };
                                    }
                                }
                            }
                            // Did not exist! Only make sure that it is acceptable
                            else
                            {
                                int notch = searchMode.Accepts(thisScanprecursorMass, candidatePeptide.MonoisotopicMassIncludingFixedMods);
                                if (notch >= 0)
                                {
                                    bestPeptides = new List<CompactPeptide> { candidatePeptide };
                                    bestScores = consideredScore;
                                    bestNotches = new List<int> { notch };
                                }
                            }
                        }
                    }
                    if (bestPeptides != null)
                    {
                        int startIndex = 0;

                        if (globalPsms[i] == null)
                        {
                            globalPsms[i] = new Psm(bestPeptides[0], bestNotches[0], bestScores, i, thisScan, CommonParameters.ExcelCompatible);
                            startIndex = 1;
                        }

                        for (int k = startIndex; k < bestPeptides.Count; k++)
                        {
                            globalPsms[i].AddOrReplace(bestPeptides[k], bestScores, bestNotches[k], CommonParameters.ReportAllAmbiguity);
                        }
                    }
                }
                lock (outputObject)
                {
                    scansSeen += fff.Item2 - fff.Item1;
                    var new_progress = (int)((double)scansSeen / (listOfSortedms2ScansLength) * 100);
                    if (new_progress > old_progress)
                    {
                        ReportProgress(new ProgressEventArgs(new_progress, "In modern search loop" + currentPartition + "/" + CommonParameters.TotalPartitions, nestedIds));
                        old_progress = new_progress;
                    }
                }
            });
            return new MetaMorpheusEngineResults(this);
        }

        protected void CalculatePeptideScores(IMsDataScan<IMzSpectrum<IMzPeak>> spectrum, double[] peptideScores, double thePrecursorMass)
        {
            //create previous variables to determine if peaks can be sequestered
            double previousTheAdd = 1 + spectrum.MassSpectrum[0].Intensity / spectrum.TotalIonCurrent;
            double previousExperimentalPeakInDaltons = spectrum.MassSpectrum[0].Mz - Constants.protonMass;
            double previousMinRange = CommonParameters.ProductMassTolerance.GetMinimumValue(previousExperimentalPeakInDaltons);
            double previousMaxRange;
            //search observed peaks
            for (int i = 1; i < spectrum.MassSpectrum.Size; i++)
            {
                double experimentalPeakInDaltons = spectrum.MassSpectrum[i].Mz - Constants.protonMass;
                if (CommonParameters.ProductMassTolerance.Within(previousExperimentalPeakInDaltons, experimentalPeakInDaltons))
                {
                    previousTheAdd += spectrum.MassSpectrum[i].Intensity / spectrum.TotalIonCurrent; //open to debate, currently sum intensities of all peaks within tolerance like it was low res
                }
                else
                {
                    previousMaxRange = CommonParameters.ProductMassTolerance.GetMaximumValue(previousExperimentalPeakInDaltons);
                    FindPeakMatches(previousTheAdd, previousMinRange, previousMaxRange, peptideScores);
                    previousTheAdd = 1 + spectrum.MassSpectrum[i].Intensity / spectrum.TotalIonCurrent;
                    previousMinRange = CommonParameters.ProductMassTolerance.GetMinimumValue(experimentalPeakInDaltons);
                }
                previousExperimentalPeakInDaltons = experimentalPeakInDaltons;
            }
            previousMaxRange = CommonParameters.ProductMassTolerance.GetMaximumValue(previousExperimentalPeakInDaltons);
            FindPeakMatches(previousTheAdd, previousMinRange, previousMaxRange, peptideScores);


            //generate experimental complementary ions if specified
            if (addCompIons)
            {
                //okay, we're not actually adding in complementary m/z peaks, we're doing a shortcut and just straight up adding the mass assuming that they're z=1
                bool HCD = lp.Contains(ProductType.B) || lp.Contains(ProductType.Y);
                bool ETD = lp.Contains(ProductType.C) || lp.Contains(ProductType.Zdot);
                IMzPeak[][] complimentaryPeaks = new IMzPeak[2][];

                if (HCD)
                {
                    complimentaryPeaks[0] = new IMzPeak[spectrum.MassSpectrum.Size];
                    for (int i = 0; i < spectrum.MassSpectrum.Size; i++)
                        complimentaryPeaks[0][i] = new MzPeak((thePrecursorMass - spectrum.MassSpectrum[i].Mz + Constants.protonMass), (spectrum.MassSpectrum[i].Intensity));
                }

                if (ETD)
                {
                    complimentaryPeaks[1] = new IMzPeak[spectrum.MassSpectrum.Size];
                    for (int i = 0; i < spectrum.MassSpectrum.Size; i++)
{'fraction_non_alphanumeric': 0.06597694890643434, 'fraction_numerical': 0.0047191215173790725, 'mean_word_length': 1.5818884723523898, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3622620', 'n_tokens_mistral': 5548, 'n_tokens_neox': 4978, 'n_words': 1135}