<commit_before>﻿using EngineLayer;
using EngineLayer.Analysis;
using MathNet.Numerics.Distributions;
using MzLibUtil;
using Nett;
using Proteomics;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using UsefulProteomicsDatabases;
using System.Xml.Serialization;
using Chemistry;

namespace TaskLayer
{
    public enum MyTask
    {
        Search,
        Gptmd,
        Calibrate
    }

    public abstract class MetaMorpheusTask
    {

        #region Public Fields

        public static readonly TomlConfig tomlConfig = TomlConfig.Create(cfg => cfg
                        .ConfigureType<Tolerance>(type => type
                            .WithConversionFor<TomlString>(convert => convert
                                .ToToml(custom => custom.ToString())
                                .FromToml(tmlString => new Tolerance(tmlString.Value))))
                        .ConfigureType<SearchMode>(type => type
                            .WithConversionFor<TomlString>(convert => convert
                                .ToToml(custom => custom.ToString())
                                .FromToml(tmlString => MetaMorpheusTask.ParseSearchMode(tmlString.Value))))
                        .ConfigureType<Protease>(type => type
                            .WithConversionFor<TomlString>(convert => convert
                                .ToToml(custom => custom.ToString())
                                .FromToml(tmlString => GlobalTaskLevelSettings.ProteaseDictionary[tmlString.Value])))
                        .ConfigureType<List<Tuple<string, string>>>(type => type
                             .WithConversionFor<TomlString>(convert => convert
                                 .ToToml(custom => string.Join("\t\t", custom.Select(b => b.Item1 + "\t" + b.Item2)))
                                 .FromToml(tmlString => GetModsFromString(tmlString.Value)))));

        private static List<Tuple<string, string>> GetModsFromString(string value)
        {
            return value.Split(new string[] { "\t\t" }, StringSplitOptions.None).Select(b => new Tuple<string, string>(b.Split('\t').First(), b.Split('\t').Last())).ToList();
        }

        #endregion Public Fields

        #region Protected Fields

        protected MyTaskResults myTaskResults;

        #endregion Protected Fields

        #region Public Constructors

        public MetaMorpheusTask(MyTask taskType)
        {
            this.TaskType = taskType;
        }

        #endregion Public Constructors

        #region Public Events

        public static event EventHandler<SingleTaskEventArgs> FinishedSingleTaskHandler;

        public static event EventHandler<SingleFileEventArgs> FinishedWritingFileHandler;

        public static event EventHandler<SingleTaskEventArgs> StartingSingleTaskHander;

        public static event EventHandler<StringEventArgs> StartingDataFileHandler;

        public static event EventHandler<StringEventArgs> FinishedDataFileHandler;

        public static event EventHandler<StringEventArgs> OutLabelStatusHandler;

        public static event EventHandler<StringEventArgs> WarnHandler;

        public static event EventHandler<StringEventArgs> NewCollectionHandler;

        public static event EventHandler<ProgressEventArgs> OutProgressHandler;

        #endregion Public Events

        #region Public Properties

        public MyTask TaskType { get; set; }

        #endregion Public Properties

        #region Public Methods

        public static SearchMode ParseSearchMode(string text)
        {
            SearchMode ye = null;

            var split = text.Split(' ');

            switch (split[1])
            {
                case "dot":
                    ToleranceUnit tu = ToleranceUnit.PPM;
                    if (split[3].ToUpperInvariant().Equals("PPM"))
                        tu = ToleranceUnit.PPM;
                    else if (split[3].ToUpperInvariant().Equals("DA"))
                        tu = ToleranceUnit.Absolute;
                    else
                        break;

                    var massShifts = Array.ConvertAll(split[4].Split(','), Double.Parse);
                    var newString = split[2].Replace("±", "");
                    var toleranceValue = double.Parse(newString, CultureInfo.InvariantCulture);
                    ye = new DotSearchMode(split[0], massShifts, new Tolerance(tu, toleranceValue));
                    break;

                case "interval":
                    IEnumerable<DoubleRange> doubleRanges = Array.ConvertAll(split[2].Split(','), b => new DoubleRange(double.Parse(b.Trim(new char[] { '[', ']' }).Split(';')[0], CultureInfo.InvariantCulture), double.Parse(b.Trim(new char[] { '[', ']' }).Split(';')[1], CultureInfo.InvariantCulture)));
                    ye = new IntervalSearchMode(split[0], doubleRanges);
                    break;

                case "OpenSearch":
                    ye = new OpenSearchMode();
                    break;

                case "daltonsAroundZero":
                    ye = new SingleAbsoluteAroundZeroSearchMode(double.Parse(split[2], CultureInfo.InvariantCulture));
                    break;

                case "ppmAroundZero":
                    ye = new SinglePpmAroundZeroSearchMode(double.Parse(split[2], CultureInfo.InvariantCulture));
                    break;

                default:
                    throw new Exception("Could not parse search mode string");
            }
            return ye;
        }

        public MyTaskResults RunTask(string output_folder, List<DbForTask> currentXmlDbFilenameList, List<string> currentRawDataFilenameList, string taskId)
        {
            StartingSingleTask(taskId);
            var paramsFileName = Path.Combine(output_folder, "prose.txt");
            using (StreamWriter file = new StreamWriter(paramsFileName))
            {
                file.WriteLine("MetaMorpheus version "
                    + (GlobalEngineLevelSettings.MetaMorpheusVersion.Equals("1.0.0.0") ? "NOT A RELEASE" : GlobalEngineLevelSettings.MetaMorpheusVersion)
                    + " is used to run a "
                    + this.TaskType
                    + " task on "
                    + currentRawDataFilenameList.Count
                    + " spectra files.");

                file.WriteLine(ToString());

                file.WriteLine();
                file.WriteLine("taskId: " + taskId);
                file.WriteLine("Spectra files:");
                file.WriteLine(string.Join(Environment.NewLine, currentRawDataFilenameList.Select(b => '\t' + b)));
                file.WriteLine("XML files:");
                file.Write(string.Join(Environment.NewLine, currentXmlDbFilenameList.Select(b => '\t' + (b.IsContaminant ? "Contaminant " : "") + b.FileName)));
            }
            SucessfullyFinishedWritingFile(paramsFileName, new List<string> { taskId });

            // TOML
            var tomlFileName = Path.Combine(output_folder, GetType().Name + "config.toml");

            Toml.WriteFile(this, tomlFileName, tomlConfig);
            SucessfullyFinishedWritingFile(tomlFileName, new List<string> { taskId });

            MetaMorpheusEngine.FinishedSingleEngineHandler += SingleEngineHandlerInTask;

#if !DEBUG
            try
            {
#endif
            var stopWatch = new Stopwatch();
            stopWatch.Start();
            RunSpecific(output_folder, currentXmlDbFilenameList, currentRawDataFilenameList, taskId);
            stopWatch.Stop();
            myTaskResults.Time = stopWatch.Elapsed;
            var resultsFileName = Path.Combine(output_folder, "results.txt");
            using (StreamWriter file = new StreamWriter(resultsFileName))
            {
                file.WriteLine(GlobalEngineLevelSettings.MetaMorpheusVersion.Equals("1.0.0.0") ? "MetaMorpheus: Not a release version" : "MetaMorpheus: version " + GlobalEngineLevelSettings.MetaMorpheusVersion);
                file.Write(myTaskResults.ToString());
            }
            SucessfullyFinishedWritingFile(resultsFileName, new List<string> { taskId });
            FinishedSingleTask(taskId);
#if !DEBUG
            }
            catch (Exception e)
            {
                MetaMorpheusEngine.FinishedSingleEngineHandler -= SingleEngineHandlerInTask;
                var resultsFileName = Path.Combine(output_folder, "results.txt");
                using (StreamWriter file = new StreamWriter(resultsFileName))
                {
                    file.WriteLine(GlobalEngineLevelSettings.MetaMorpheusVersion.Equals("1.0.0.0") ? "MetaMorpheus: Not a release version" : "MetaMorpheus: version " + GlobalEngineLevelSettings.MetaMorpheusVersion);
                    file.Write("e: " + e);
                    file.Write("e.Message: " + e.Message);
                    file.Write("e.InnerException: " + e.InnerException);
                    file.Write("e.Source: " + e.Source);
                    file.Write("e.StackTrace: " + e.StackTrace);
                    file.Write("e.TargetSite: " + e.TargetSite);
                }
                throw;
            }
#endif

            MetaMorpheusEngine.FinishedSingleEngineHandler -= SingleEngineHandlerInTask;
            return myTaskResults;
        }

        #endregion Public Methods

        #region Protected Internal Methods

        protected internal void WritePsmsToTsv(List<NewPsmWithFdr> items, string outputFolder, string fileName, List<string> nestedIds)
        {
            var writtenFile = Path.Combine(outputFolder, fileName + ".psmtsv");
            using (StreamWriter output = new StreamWriter(writtenFile))
            {
                output.WriteLine(NewPsmWithFdr.TabSeparatedHeader);
                for (int i = 0; i < items.Count; i++)
                    output.WriteLine(items[i]);
            }
            SucessfullyFinishedWritingFile(writtenFile, nestedIds);
        }

        protected internal void WriteMzidentml(List<NewPsmWithFdr> items, List<ProteinGroup> groups, List<ModificationWithMass> variableMods, List<ModificationWithMass> fixedMods, List<Protease> proteases, double threshold, SearchMode searchMode, Tolerance productTolerance, int missedCleavages, string outputFolder, string fileName, List<string> nestedIds)
        {
            List<PeptideWithSetModifications> peptides = items.SelectMany(i => i.thisPSM.Pli.PeptidesWithSetModifications).Distinct().ToList();
            List<Protein> proteins = peptides.Select(p => p.Protein).Distinct().ToList();
            List<string> filenames = items.Select(i => i.thisPSM.FileName).Distinct().ToList();

            XmlSerializer _indexedSerializer = new XmlSerializer(typeof(mzIdentML.Generated.MzIdentMLType));
            var _mzid = new mzIdentML.Generated.MzIdentMLType()
            {
                version = "1.1.1",
            };

            //cvlist: URLs of controlled vocabularies used within the file.
            _mzid.cvList = new mzIdentML.Generated.cvType[2] { new mzIdentML.Generated.cvType()
            {
                id = "PSI-MS",
                fullName = "Proteomics Standards Initiative Mass Spectrometry Vocabularies",
                uri = "https://github.com/HUPO-PSI/psi-ms-CV/blob/master/psi-ms.obo",
                version= "4.0.9"
            },
            new mzIdentML.Generated.cvType()
            {
                id = "UO",
                fullName = "UNIT-ONTOLOGY",
                uri = "http://www.unimod.org/obo/unimod.obo"
            }
            };

            _mzid.AnalysisSoftwareList = new mzIdentML.Generated.AnalysisSoftwareType[1] {  new mzIdentML.Generated.AnalysisSoftwareType()
            {
                id = "AS_MetaMorpheus",
                name = "MetaMorpheus",
                version = GlobalEngineLevelSettings.MetaMorpheusVersion,
                uri = "https://github.com/smith-chem-wisc/MetaMorpheus",
                SoftwareName = new mzIdentML.Generated.ParamType()
                {
                    Item = new mzIdentML.Generated.CVParamType()
                    {
                        //using Morpheus's accession until we get MetaMorpheus entered
                        accession = "MS:1002661",
                        name = "Morpheus",
                        cvRef = "PSI-MS"
                    }
                }
            }};
            _mzid.SequenceCollection = new mzIdentML.Generated.SequenceCollectionType()
            {
                Peptide = new mzIdentML.Generated.PeptideType[peptides.Count()],
                DBSequence = new mzIdentML.Generated.DBSequenceType[proteins.Count()],
                PeptideEvidence = new mzIdentML.Generated.PeptideEvidenceType[peptides.Count()]
            };
            int protein_index = 0;
            foreach (Protein protein in proteins)
            {
                _mzid.SequenceCollection.DBSequence[protein_index] = new mzIdentML.Generated.DBSequenceType()
                {
                    id = "DBS_" + protein.Accession,
                    lengthSpecified = true,
                    length = protein.Length,
                    searchDatabase_ref = "SDB_1", //TODO: SPECIFIC DATABASE THE PROTEIN CAME FROM? NULL FOR FOR PROTEINS
                    accession = protein.Accession,
                    Seq = protein.BaseSequence,
                    cvParam = new mzIdentML.Generated.CVParamType[1]
                    {
                        new mzIdentML.Generated.CVParamType()
                        {
                            accession = "MS:1001088",
                            name = "protein description",
                            cvRef = "PSI-MS",
                            value = protein.FullDescription
                        }
                    },
                    name = protein.Name
                };
                protein_index++;
            }

            _mzid.DataCollection = new mzIdentML.Generated.DataCollectionType()
            {

                AnalysisData = new mzIdentML.Generated.AnalysisDataType()
                {
                    SpectrumIdentificationList = new mzIdentML.Generated.SpectrumIdentificationListType[1]
                    {
                        new mzIdentML.Generated.SpectrumIdentificationListType()
                        {
                            id = "SIL",
                            SpectrumIdentificationResult = new mzIdentML.Generated.SpectrumIdentificationResultType[items.Count]
                        }
                    }
                },
                Inputs = new mzIdentML.Generated.InputsType()
                {
                    //TODO: SEPARATE SEARCH DATABASES IF MULTIPLE DATABASES ENTERED
                    SearchDatabase = new mzIdentML.Generated.SearchDatabaseType[1]
                    {
                        new mzIdentML.Generated.SearchDatabaseType()
                        {
                         id = "SDB_1",
                            FileFormat = new mzIdentML.Generated.FileFormatType()
                            {
                                cvParam = new mzIdentML.Generated.CVParamType()
                                {
                                    accession = "MS:1002660",
                                    name = "UniProtKB XML sequence format",
                                    cvRef = "PSI-MS"
                                }
                            },
                            DatabaseName = new mzIdentML.Generated.ParamType()
                            {
                                Item = new mzIdentML.Generated.CVParamType()
                                {
                                    accession = "MS:1001073",
                                    name = "database type amino acid",
                                    cvRef = "PSI-MS"
                                }
                            },
                        }
                    },
                    SpectraData = new mzIdentML.Generated.SpectraDataType[filenames.Count]
                }
            };

            _mzid.AnalysisCollection = new mzIdentML.Generated.AnalysisCollectionType()
            {
                SpectrumIdentification = new mzIdentML.Generated.SpectrumIdentificationType[1]
                {
                    new mzIdentML.Generated.SpectrumIdentificationType()
                    {
                        id = "SI",
                        spectrumIdentificationList_ref = "SIL",
                        spectrumIdentificationProtocol_ref = "SIP",
                        InputSpectra = new mzIdentML.Generated.InputSpectraType[filenames.Count],
                        SearchDatabaseRef = new mzIdentML.Generated.SearchDatabaseRefType[proteins.Count]
                    }
                }
            };

            Dictionary<string, int> spectral_ids = new Dictionary<string, int>(); //key is datafile, value is datafile's id
            int spectra_data_id = 0;
            foreach (string data_filepath in filenames)
            {
                bool thermoRawFile = Path.GetExtension(data_filepath) == ".raw";
                string spectral_data_id = "SD_" + spectra_data_id;
                spectral_ids.Add(data_filepath, spectra_data_id);
                _mzid.AnalysisCollection.SpectrumIdentification[0].InputSpectra[spectra_data_id] = new mzIdentML.Generated.InputSpectraType()
                {
                    spectraData_ref = spectral_data_id
                };
                _mzid.DataCollection.Inputs.SpectraData[spectra_data_id] = new mzIdentML.Generated.SpectraDataType()
                {
                    id = spectral_data_id,
                    name = Path.GetFileNameWithoutExtension(data_filepath),
                    location = Path.GetPathRoot(data_filepath),
                    FileFormat = new mzIdentML.Generated.FileFormatType()
                    {
                        cvParam = new mzIdentML.Generated.CVParamType()
                        {
                            accession = thermoRawFile ? "MS:1000563" : "MS:1000584",
                            name = thermoRawFile ?  "Thermo RAW format" : "mzML format",
                            cvRef = "PSI-MS"
                        }
                    },
                    SpectrumIDFormat = new mzIdentML.Generated.SpectrumIDFormatType()
                    {
                        cvParam = new mzIdentML.Generated.CVParamType()
                        {
                            accession = thermoRawFile? "MS:1000768" : "MS:1001530",
                            name = thermoRawFile? "Thermo nativeID format" : "mzML unique identifier",
                            cvRef = "PSI-MS"
                        }
                    }
                };
                spectra_data_id++;
            }

            int sir_id = 0;
            int pe_index = 0;
            int p_index = 0;
            Dictionary<PeptideWithSetModifications, Tuple<int, int, List<string>>> peptide_ids = new Dictionary<PeptideWithSetModifications, Tuple<int, int, List<string>>>(); //key is peptide, value is <peptide id for that peptide, peptide evidence id>
            Dictionary<Tuple<string, int>, Tuple<int, int>> psm_per_scan = new Dictionary<Tuple<string, int>, Tuple<int, int>>(); //key is <filename, scan numer> value is <scan result id, scan item id #'s (could be more than one ID per scan)>
            foreach (NewPsmWithFdr psm in items)
            {
                PeptideWithSetModifications peptide = psm.thisPSM.Pli.PeptidesWithSetModifications.OrderBy(p => p.PeptideDescription).First();
                Tuple<int, int, List<string>> peptide_id;
                //if first peptide on list hasn't been added, add peptide and peptide evidence
                if (!peptide_ids.TryGetValue(peptide, out peptide_id))
                {
                    peptide_id = new Tuple<int, int, List<string>>(p_index, 0, new List<string>());
                    p_index++;
                    _mzid.SequenceCollection.Peptide[peptide_id.Item1] = new mzIdentML.Generated.PeptideType()
                    {
                        PeptideSequence = peptide.BaseSequence,
                        id = "P_" + peptide_id.Item1,
                        Modification = new mzIdentML.Generated.ModificationType[peptide.NumMods]
                    };
                    int mod_id = 0;
                    foreach (KeyValuePair<int, ModificationWithMass> mod in peptide.allModsOneIsNterminus)
                    {
                        _mzid.SequenceCollection.Peptide[peptide_id.Item1].Modification[mod_id] = new mzIdentML.Generated.ModificationType()
                        {
                            location = mod.Key - 1,
                            locationSpecified = true,
                            monoisotopicMassDelta = mod.Value.monoisotopicMass,
                            residues = new string[1] { mod.Value.motif.ToString() },
                            monoisotopicMassDeltaSpecified = true,
                            cvParam = new mzIdentML.Generated.CVParamType[1]
                            {
                            new mzIdentML.Generated.CVParamType()
                            {
                                cvRef = "PSI-MS",
                                name = "unknown modification",<commit_msg>motif string display<commit_after>﻿using EngineLayer;
using EngineLayer.Analysis;
using MathNet.Numerics.Distributions;
using MzLibUtil;
using Nett;
using Proteomics;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using UsefulProteomicsDatabases;
using System.Xml.Serialization;
using Chemistry;

namespace TaskLayer
{
    public enum MyTask
    {
        Search,
        Gptmd,
        Calibrate
    }

    public abstract class MetaMorpheusTask
    {

        #region Public Fields

        public static readonly TomlConfig tomlConfig = TomlConfig.Create(cfg => cfg
                        .ConfigureType<Tolerance>(type => type
                            .WithConversionFor<TomlString>(convert => convert
                                .ToToml(custom => custom.ToString())
                                .FromToml(tmlString => new Tolerance(tmlString.Value))))
                        .ConfigureType<SearchMode>(type => type
                            .WithConversionFor<TomlString>(convert => convert
                                .ToToml(custom => custom.ToString())
                                .FromToml(tmlString => MetaMorpheusTask.ParseSearchMode(tmlString.Value))))
                        .ConfigureType<Protease>(type => type
                            .WithConversionFor<TomlString>(convert => convert
                                .ToToml(custom => custom.ToString())
                                .FromToml(tmlString => GlobalTaskLevelSettings.ProteaseDictionary[tmlString.Value])))
                        .ConfigureType<List<Tuple<string, string>>>(type => type
                             .WithConversionFor<TomlString>(convert => convert
                                 .ToToml(custom => string.Join("\t\t", custom.Select(b => b.Item1 + "\t" + b.Item2)))
                                 .FromToml(tmlString => GetModsFromString(tmlString.Value)))));

        private static List<Tuple<string, string>> GetModsFromString(string value)
        {
            return value.Split(new string[] { "\t\t" }, StringSplitOptions.None).Select(b => new Tuple<string, string>(b.Split('\t').First(), b.Split('\t').Last())).ToList();
        }

        #endregion Public Fields

        #region Protected Fields

        protected MyTaskResults myTaskResults;

        #endregion Protected Fields

        #region Public Constructors

        public MetaMorpheusTask(MyTask taskType)
        {
            this.TaskType = taskType;
        }

        #endregion Public Constructors

        #region Public Events

        public static event EventHandler<SingleTaskEventArgs> FinishedSingleTaskHandler;

        public static event EventHandler<SingleFileEventArgs> FinishedWritingFileHandler;

        public static event EventHandler<SingleTaskEventArgs> StartingSingleTaskHander;

        public static event EventHandler<StringEventArgs> StartingDataFileHandler;

        public static event EventHandler<StringEventArgs> FinishedDataFileHandler;

        public static event EventHandler<StringEventArgs> OutLabelStatusHandler;

        public static event EventHandler<StringEventArgs> WarnHandler;

        public static event EventHandler<StringEventArgs> NewCollectionHandler;

        public static event EventHandler<ProgressEventArgs> OutProgressHandler;

        #endregion Public Events

        #region Public Properties

        public MyTask TaskType { get; set; }

        #endregion Public Properties

        #region Public Methods

        public static SearchMode ParseSearchMode(string text)
        {
            SearchMode ye = null;

            var split = text.Split(' ');

            switch (split[1])
            {
                case "dot":
                    ToleranceUnit tu = ToleranceUnit.PPM;
                    if (split[3].ToUpperInvariant().Equals("PPM"))
                        tu = ToleranceUnit.PPM;
                    else if (split[3].ToUpperInvariant().Equals("DA"))
                        tu = ToleranceUnit.Absolute;
                    else
                        break;

                    var massShifts = Array.ConvertAll(split[4].Split(','), Double.Parse);
                    var newString = split[2].Replace("±", "");
                    var toleranceValue = double.Parse(newString, CultureInfo.InvariantCulture);
                    ye = new DotSearchMode(split[0], massShifts, new Tolerance(tu, toleranceValue));
                    break;

                case "interval":
                    IEnumerable<DoubleRange> doubleRanges = Array.ConvertAll(split[2].Split(','), b => new DoubleRange(double.Parse(b.Trim(new char[] { '[', ']' }).Split(';')[0], CultureInfo.InvariantCulture), double.Parse(b.Trim(new char[] { '[', ']' }).Split(';')[1], CultureInfo.InvariantCulture)));
                    ye = new IntervalSearchMode(split[0], doubleRanges);
                    break;

                case "OpenSearch":
                    ye = new OpenSearchMode();
                    break;

                case "daltonsAroundZero":
                    ye = new SingleAbsoluteAroundZeroSearchMode(double.Parse(split[2], CultureInfo.InvariantCulture));
                    break;

                case "ppmAroundZero":
                    ye = new SinglePpmAroundZeroSearchMode(double.Parse(split[2], CultureInfo.InvariantCulture));
                    break;

                default:
                    throw new Exception("Could not parse search mode string");
            }
            return ye;
        }

        public MyTaskResults RunTask(string output_folder, List<DbForTask> currentXmlDbFilenameList, List<string> currentRawDataFilenameList, string taskId)
        {
            StartingSingleTask(taskId);
            var paramsFileName = Path.Combine(output_folder, "prose.txt");
            using (StreamWriter file = new StreamWriter(paramsFileName))
            {
                file.WriteLine("MetaMorpheus version "
                    + (GlobalEngineLevelSettings.MetaMorpheusVersion.Equals("1.0.0.0") ? "NOT A RELEASE" : GlobalEngineLevelSettings.MetaMorpheusVersion)
                    + " is used to run a "
                    + this.TaskType
                    + " task on "
                    + currentRawDataFilenameList.Count
                    + " spectra files.");

                file.WriteLine(ToString());

                file.WriteLine();
                file.WriteLine("taskId: " + taskId);
                file.WriteLine("Spectra files:");
                file.WriteLine(string.Join(Environment.NewLine, currentRawDataFilenameList.Select(b => '\t' + b)));
                file.WriteLine("XML files:");
                file.Write(string.Join(Environment.NewLine, currentXmlDbFilenameList.Select(b => '\t' + (b.IsContaminant ? "Contaminant " : "") + b.FileName)));
            }
            SucessfullyFinishedWritingFile(paramsFileName, new List<string> { taskId });

            // TOML
            var tomlFileName = Path.Combine(output_folder, GetType().Name + "config.toml");

            Toml.WriteFile(this, tomlFileName, tomlConfig);
            SucessfullyFinishedWritingFile(tomlFileName, new List<string> { taskId });

            MetaMorpheusEngine.FinishedSingleEngineHandler += SingleEngineHandlerInTask;

#if !DEBUG
            try
            {
#endif
            var stopWatch = new Stopwatch();
            stopWatch.Start();
            RunSpecific(output_folder, currentXmlDbFilenameList, currentRawDataFilenameList, taskId);
            stopWatch.Stop();
            myTaskResults.Time = stopWatch.Elapsed;
            var resultsFileName = Path.Combine(output_folder, "results.txt");
            using (StreamWriter file = new StreamWriter(resultsFileName))
            {
                file.WriteLine(GlobalEngineLevelSettings.MetaMorpheusVersion.Equals("1.0.0.0") ? "MetaMorpheus: Not a release version" : "MetaMorpheus: version " + GlobalEngineLevelSettings.MetaMorpheusVersion);
                file.Write(myTaskResults.ToString());
            }
            SucessfullyFinishedWritingFile(resultsFileName, new List<string> { taskId });
            FinishedSingleTask(taskId);
#if !DEBUG
            }
            catch (Exception e)
            {
                MetaMorpheusEngine.FinishedSingleEngineHandler -= SingleEngineHandlerInTask;
                var resultsFileName = Path.Combine(output_folder, "results.txt");
                using (StreamWriter file = new StreamWriter(resultsFileName))
                {
                    file.WriteLine(GlobalEngineLevelSettings.MetaMorpheusVersion.Equals("1.0.0.0") ? "MetaMorpheus: Not a release version" : "MetaMorpheus: version " + GlobalEngineLevelSettings.MetaMorpheusVersion);
                    file.Write("e: " + e);
                    file.Write("e.Message: " + e.Message);
                    file.Write("e.InnerException: " + e.InnerException);
                    file.Write("e.Source: " + e.Source);
                    file.Write("e.StackTrace: " + e.StackTrace);
                    file.Write("e.TargetSite: " + e.TargetSite);
                }
                throw;
            }
#endif

            MetaMorpheusEngine.FinishedSingleEngineHandler -= SingleEngineHandlerInTask;
            return myTaskResults;
        }

        #endregion Public Methods

        #region Protected Internal Methods

        protected internal void WritePsmsToTsv(List<NewPsmWithFdr> items, string outputFolder, string fileName, List<string> nestedIds)
        {
            var writtenFile = Path.Combine(outputFolder, fileName + ".psmtsv");
            using (StreamWriter output = new StreamWriter(writtenFile))
            {
                output.WriteLine(NewPsmWithFdr.TabSeparatedHeader);
                for (int i = 0; i < items.Count; i++)
                    output.WriteLine(items[i]);
            }
            SucessfullyFinishedWritingFile(writtenFile, nestedIds);
        }

        protected internal void WriteMzidentml(List<NewPsmWithFdr> items, List<ProteinGroup> groups, List<ModificationWithMass> variableMods, List<ModificationWithMass> fixedMods, List<Protease> proteases, double threshold, SearchMode searchMode, Tolerance productTolerance, int missedCleavages, string outputFolder, string fileName, List<string> nestedIds)
        {
            List<PeptideWithSetModifications> peptides = items.SelectMany(i => i.thisPSM.Pli.PeptidesWithSetModifications).Distinct().ToList();
            List<Protein> proteins = peptides.Select(p => p.Protein).Distinct().ToList();
            List<string> filenames = items.Select(i => i.thisPSM.FileName).Distinct().ToList();

            XmlSerializer _indexedSerializer = new XmlSerializer(typeof(mzIdentML.Generated.MzIdentMLType));
            var _mzid = new mzIdentML.Generated.MzIdentMLType()
            {
                version = "1.1.1",
            };

            //cvlist: URLs of controlled vocabularies used within the file.
            _mzid.cvList = new mzIdentML.Generated.cvType[2] { new mzIdentML.Generated.cvType()
            {
                id = "PSI-MS",
                fullName = "Proteomics Standards Initiative Mass Spectrometry Vocabularies",
                uri = "https://github.com/HUPO-PSI/psi-ms-CV/blob/master/psi-ms.obo",
                version= "4.0.9"
            },
            new mzIdentML.Generated.cvType()
            {
                id = "UO",
                fullName = "UNIT-ONTOLOGY",
                uri = "http://www.unimod.org/obo/unimod.obo"
            }
            };

            _mzid.AnalysisSoftwareList = new mzIdentML.Generated.AnalysisSoftwareType[1] {  new mzIdentML.Generated.AnalysisSoftwareType()
            {
                id = "AS_MetaMorpheus",
                name = "MetaMorpheus",
                version = GlobalEngineLevelSettings.MetaMorpheusVersion,
                uri = "https://github.com/smith-chem-wisc/MetaMorpheus",
                SoftwareName = new mzIdentML.Generated.ParamType()
                {
                    Item = new mzIdentML.Generated.CVParamType()
                    {
                        //using Morpheus's accession until we get MetaMorpheus entered
                        accession = "MS:1002661",
                        name = "Morpheus",
                        cvRef = "PSI-MS"
                    }
                }
            }};
            _mzid.SequenceCollection = new mzIdentML.Generated.SequenceCollectionType()
            {
                Peptide = new mzIdentML.Generated.PeptideType[peptides.Count()],
                DBSequence = new mzIdentML.Generated.DBSequenceType[proteins.Count()],
                PeptideEvidence = new mzIdentML.Generated.PeptideEvidenceType[peptides.Count()]
            };
            int protein_index = 0;
            foreach (Protein protein in proteins)
            {
                _mzid.SequenceCollection.DBSequence[protein_index] = new mzIdentML.Generated.DBSequenceType()
                {
                    id = "DBS_" + protein.Accession,
                    lengthSpecified = true,
                    length = protein.Length,
                    searchDatabase_ref = "SDB_1", //TODO: SPECIFIC DATABASE THE PROTEIN CAME FROM? NULL FOR FOR PROTEINS
                    accession = protein.Accession,
                    Seq = protein.BaseSequence,
                    cvParam = new mzIdentML.Generated.CVParamType[1]
                    {
                        new mzIdentML.Generated.CVParamType()
                        {
                            accession = "MS:1001088",
                            name = "protein description",
                            cvRef = "PSI-MS",
                            value = protein.FullDescription
                        }
                    },
                    name = protein.Name
                };
                protein_index++;
            }

            _mzid.DataCollection = new mzIdentML.Generated.DataCollectionType()
            {

                AnalysisData = new mzIdentML.Generated.AnalysisDataType()
                {
                    SpectrumIdentificationList = new mzIdentML.Generated.SpectrumIdentificationListType[1]
                    {
                        new mzIdentML.Generated.SpectrumIdentificationListType()
                        {
                            id = "SIL",
                            SpectrumIdentificationResult = new mzIdentML.Generated.SpectrumIdentificationResultType[items.Count]
                        }
                    }
                },
                Inputs = new mzIdentML.Generated.InputsType()
                {
                    //TODO: SEPARATE SEARCH DATABASES IF MULTIPLE DATABASES ENTERED
                    SearchDatabase = new mzIdentML.Generated.SearchDatabaseType[1]
                    {
                        new mzIdentML.Generated.SearchDatabaseType()
                        {
                         id = "SDB_1",
                            FileFormat = new mzIdentML.Generated.FileFormatType()
                            {
                                cvParam = new mzIdentML.Generated.CVParamType()
                                {
                                    accession = "MS:1002660",
                                    name = "UniProtKB XML sequence format",
                                    cvRef = "PSI-MS"
                                }
                            },
                            DatabaseName = new mzIdentML.Generated.ParamType()
                            {
                                Item = new mzIdentML.Generated.CVParamType()
                                {
                                    accession = "MS:1001073",
                                    name = "database type amino acid",
                                    cvRef = "PSI-MS"
                                }
                            },
                        }
                    },
                    SpectraData = new mzIdentML.Generated.SpectraDataType[filenames.Count]
                }
            };

            _mzid.AnalysisCollection = new mzIdentML.Generated.AnalysisCollectionType()
            {
                SpectrumIdentification = new mzIdentML.Generated.SpectrumIdentificationType[1]
                {
                    new mzIdentML.Generated.SpectrumIdentificationType()
                    {
                        id = "SI",
                        spectrumIdentificationList_ref = "SIL",
                        spectrumIdentificationProtocol_ref = "SIP",
                        InputSpectra = new mzIdentML.Generated.InputSpectraType[filenames.Count],
                        SearchDatabaseRef = new mzIdentML.Generated.SearchDatabaseRefType[proteins.Count]
                    }
                }
            };

            Dictionary<string, int> spectral_ids = new Dictionary<string, int>(); //key is datafile, value is datafile's id
            int spectra_data_id = 0;
            foreach (string data_filepath in filenames)
            {
                bool thermoRawFile = Path.GetExtension(data_filepath) == ".raw";
                string spectral_data_id = "SD_" + spectra_data_id;
                spectral_ids.Add(data_filepath, spectra_data_id);
                _mzid.AnalysisCollection.SpectrumIdentification[0].InputSpectra[spectra_data_id] = new mzIdentML.Generated.InputSpectraType()
                {
                    spectraData_ref = spectral_data_id
                };
                _mzid.DataCollection.Inputs.SpectraData[spectra_data_id] = new mzIdentML.Generated.SpectraDataType()
                {
                    id = spectral_data_id,
                    name = Path.GetFileNameWithoutExtension(data_filepath),
                    location = Path.GetPathRoot(data_filepath),
                    FileFormat = new mzIdentML.Generated.FileFormatType()
                    {
                        cvParam = new mzIdentML.Generated.CVParamType()
                        {
                            accession = thermoRawFile ? "MS:1000563" : "MS:1000584",
                            name = thermoRawFile ?  "Thermo RAW format" : "mzML format",
                            cvRef = "PSI-MS"
                        }
                    },
                    SpectrumIDFormat = new mzIdentML.Generated.SpectrumIDFormatType()
                    {
                        cvParam = new mzIdentML.Generated.CVParamType()
                        {
                            accession = thermoRawFile? "MS:1000768" : "MS:1001530",
                            name = thermoRawFile? "Thermo nativeID format" : "mzML unique identifier",
                            cvRef = "PSI-MS"
                        }
                    }
                };
                spectra_data_id++;
            }

            int sir_id = 0;
            int pe_index = 0;
            int p_index = 0;
            Dictionary<PeptideWithSetModifications, Tuple<int, int, List<string>>> peptide_ids = new Dictionary<PeptideWithSetModifications, Tuple<int, int, List<string>>>(); //key is peptide, value is <peptide id for that peptide, peptide evidence id>
            Dictionary<Tuple<string, int>, Tuple<int, int>> psm_per_scan = new Dictionary<Tuple<string, int>, Tuple<int, int>>(); //key is <filename, scan numer> value is <scan result id, scan item id #'s (could be more than one ID per scan)>
            foreach (NewPsmWithFdr psm in items)
            {
                PeptideWithSetModifications peptide = psm.thisPSM.Pli.PeptidesWithSetModifications.OrderBy(p => p.PeptideDescription).First();
                Tuple<int, int, List<string>> peptide_id;
                //if first peptide on list hasn't been added, add peptide and peptide evidence
                if (!peptide_ids.TryGetValue(peptide, out peptide_id))
                {
                    peptide_id = new Tuple<int, int, List<string>>(p_index, 0, new List<string>());
                    p_index++;
                    _mzid.SequenceCollection.Peptide[peptide_id.Item1] = new mzIdentML.Generated.PeptideType()
                    {
                        PeptideSequence = peptide.BaseSequence,
                        id = "P_" + peptide_id.Item1,
                        Modification = new mzIdentML.Generated.ModificationType[peptide.NumMods]
                    };
                    int mod_id = 0;
                    foreach (KeyValuePair<int, ModificationWithMass> mod in peptide.allModsOneIsNterminus)
                    {
                        _mzid.SequenceCollection.Peptide[peptide_id.Item1].Modification[mod_id] = new mzIdentML.Generated.ModificationType()
                        {
                            location = mod.Key - 1,
                            locationSpecified = true,
                            monoisotopicMassDelta = mod.Value.monoisotopicMass,
                            residues = new string[1] { mod.Value.motif.Motif.ToString() },
                            monoisotopicMassDeltaSpecified = true,
                            cvParam = new mzIdentML.Generated.CVParamType[1]
                            {
                            new mzIdentML.Generated.CVParamType()
                            {
                                cvRef = "PSI-MS",
                                name = "unknown modification",
{'fraction_non_alphanumeric': 0.08758731864588931, 'fraction_numerical': 0.005186552344461837, 'mean_word_length': 1.5757612227704898, 'pattern_counts': {'":': 10, '<': 129, '<?xml version=': 0, '>': 179, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5691162', 'n_tokens_mistral': 11200, 'n_tokens_neox': 10284, 'n_words': 1983}