<commit_before>/*
 * Copyright 2015 <NAME> <<EMAIL> at <EMAIL>>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.rappsilber.fdr;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.sql.SQLException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.rappsilber.data.csv.CsvParser;
import org.rappsilber.fdr.entities.PSM;
import org.rappsilber.utils.AutoIncrementValueMap;
import static org.rappsilber.utils.StringUtils.sixDigits;
import org.rappsilber.utils.UpdatableChar;

/**
 *
 * @author lfischer
 */
public class CSVinFDR extends OfflineFDR {
    private String m_source = null;
    private ArrayList<String[]> commandLineColumnMapping;
    private Character delimiter;
    private Character quote;
    private String[][] defaultcolumnmapping=new String[][]{
        new String[]{"peptide1","Peptide1"},
        new String[]{"peptide2","Peptide2"},
        new String[]{"peptide length 1","LengthPeptide1"},
        new String[]{"peptide length 2","LengthPeptide2"},
        new String[]{"peptide link 1","Link1"},
        new String[]{"peptide link 2","Link2"},
        new String[]{"is decoy 1","Protein1decoy"},
        new String[]{"is decoy 2","Protein2decoy"},
        new String[]{"precursor charge","PrecoursorCharge"},
        new String[]{"score","match score"},
        new String[]{"accession1","Protein1"},
        new String[]{"accession2","Protein2"},
        new String[]{"description1","Fasta1"},
        new String[]{"description2","Fasta2"},
        new String[]{"peptide position 1","Start1"},
        new String[]{"peptide position 2","Start2"},
        new String[]{"peptide1 score","Pep1Score"},
        new String[]{"peptide2 score","Pep2Score"},
        new String[]{"experimental mz","expMZ"},
        new String[]{"calculated mass","calcMass"},
        new String[]{"info","info"},
    };
    
    public CSVinFDR() {
    }

    public CSVinFDR(int[] peptideLengthGroups) {
        super(peptideLengthGroups);
    }
    

    
    @Override
    protected ArrayList<String> getPSMHeader() {
        ArrayList<String> ret = super.getPSMHeader();
//        String header = "PSMID" + seperator + "run" + seperator + "scan" 
//                + seperator + "exp charge"  + seperator + "exp m/z" + seperator + "exp mass" + seperator + "exp fractionalmass" 
//                + seperator + "match charge" + seperator + "match mass" + seperator + "match fractionalmass"   
//                
//                + seperator + "Protein1" + seperator + "Description1" + seperator + "Decoy1" + seperator + "Protein2" + seperator + "Description2" + seperator + "Decoy2" + seperator + "Peptide1" + seperator + "Peptide2" + seperator + "PeptidePosition1" + seperator + "PeptidePosition2" + seperator + "PeptideLength1" + seperator + "PeptideLength2" + seperator + "fromSite" + seperator + "ToSite" + seperator + "Charge" + seperator + "Score" + seperator + "isDecoy" + seperator + "isTT" + seperator + "isTD" + seperator + "isDD" + seperator + "fdrGroup" + seperator + "fdr" + seperator + "" + seperator + "PeptidePairFDR" + seperator + "Protein1FDR" + seperator + "Protein2FDR" + seperator + "LinkFDR" + seperator + "PPIFDR" + seperator + seperator + "peptide pair id" + seperator + "link id" + seperator + "ppi id";
        ret.add(2, "exp charge");
        ret.add(3, "exp m/z");
        ret.add(4, "exp mass");
        ret.add(5, "exp fractionalmass");
        ret.add(6, "match charge");
        ret.add(7,  "match mass");
        ret.add(8, "match fractionalmass");
        return ret;
    }

    
    protected ArrayList<String> getPSMOutputLine(PSM pp) {
        ArrayList<String> ret = super.getPSMOutputLine(pp);

        double mz = pp.getExperimentalMZ();
        int charge = pp.getCharge();
        double mass = (mz-1.00727646677)*charge;
        double fraction = mass-Math.floor(mass);
        double calcfraction = pp.getCalcMass()-Math.floor(pp.getCalcMass());

        
        ret.add(2,""+ charge);
        ret.add(3, sixDigits.format(mz));
        ret.add(4, sixDigits.format(mass));
        ret.add(5, sixDigits.format(fraction));
        ret.add(6,""+ pp.getMatchedCharge());
        ret.add(7,  sixDigits.format(pp.getCalcMass()));
        ret.add(8, sixDigits.format(calcfraction));
        

        return ret;
    }    
    

    public void readCSV(File f) throws FileNotFoundException, IOException, ParseException  {
        readCSV(CsvParser.guessCsv(f, 50));
    }

    public void readCSV(CsvParser csv) throws FileNotFoundException, IOException, ParseException  {
        if (csv.getInputFile()!= null)
            m_source = csv.getInputFile().getAbsolutePath();
            

        Integer crun = csv.getColumn("run");
        Integer cscan = csv.getColumn("scan");
        Integer cpsmID = csv.getColumn("psmid");
        int cpep1 = csv.getColumn("peptide1");
        int cpep2 = csv.getColumn("peptide2");
        Integer cpep1len = csv.getColumn("peptide length 1");
        Integer cpep2len = csv.getColumn("peptide length 2");
        int cpep1site = csv.getColumn("peptide link 1");
        int cpep2site = csv.getColumn("peptide link 2");
        int cpep1decoy = csv.getColumn("is decoy 1");
        Integer cpep2decoy = csv.getColumn("is decoy 2");
        int cprecZ = csv.getColumn("precursor charge");
        int cscore = csv.getColumn("score");
        int caccession1 = csv.getColumn("accession1");
        int caccession2 = csv.getColumn("accession2");
        Integer cdescription1 = csv.getColumn("description1");
        Integer cdescription2 = csv.getColumn("description2");
        Integer cpeptide_position1 = csv.getColumn("peptide position 1");
        Integer cpeptide_position2 = csv.getColumn("peptide position 2");
        Integer cscoreratio = csv.getColumn("score ratio");
        Integer cPepScore1 = csv.getColumn("peptide1 score");
        Integer cPepScore2 = csv.getColumn("peptide2 score");
        Integer cCrosslinker = csv.getColumn("crosslinker");
        Integer cExpMZ = csv.getColumn("experimental mz");
        Integer cCalcMass = csv.getColumn("calculated mass");
        Integer cInfo = csv.getColumn("info");
        
        int noID = 0;
        AutoIncrementValueMap<String> PSMIDs = new AutoIncrementValueMap<String>();
        int lineNumber =0;
        
        int noPSMID =0;
        while (csv.next()) {
            lineNumber++;
            String psmID;

            String pepSeq1 = csv.getValue(cpep1);
            String pepSeq2 = csv.getValue(cpep2);

            // if the sequence looks like K.PEPTIDEK.A assume that the first and 
            // last aminoacid are leading and trailing aminoacids and not really part of the peptide
            if (pepSeq1.matches("[A-Za-z\\-]+\\..*\\.[A-Za-z\\-]+")) {
                pepSeq1 = pepSeq1.replaceAll("^[A-Z\\-]+\\.", "").replaceAll("\\.[A-Z\\-]+", "");
            }
            if (pepSeq2.matches("[A-Za-z\\-]+\\..*\\.[A-Za-z\\-]+")) {
                pepSeq2 = pepSeq2.replaceAll("^[A-Z\\-]+\\.", "").replaceAll("\\.[A-Z\\-]+", "");
            }

            
            Integer site1 = csv.getInteger(cpep1site,-1);
            Integer site2 = csv.getInteger(cpep2site,-1); //pepSeq2 == null || pepSeq2.trim().isEmpty() ? -1 : csv.getInteger(cpep2site,-1);
            
            // do we have to generate an ID?
            if (cpsmID == null) {
                if (cscan == null || crun == null) {
                    psmID = Integer.toString(noPSMID++);
                } else {
                    String key = "Scan: " + csv.getValue(cscan) + " Run: " + csv.getValue(crun);
                    int c= pepSeq1.compareTo(pepSeq2) ;
                    if (c > 0 || (c==0 && site1 > site2) ) {
                        key=key +" P1_" + csv.getValue(cpep1) + " P2_" + csv.getValue(cpep2) + " " + csv.getInteger(cpep1site) + " " + csv.getInteger(cpep2site);
                    } else {
                        key=key +" P1_" + csv.getValue(cpep2) + " P2_" + csv.getValue(cpep1) + " " + csv.getInteger(cpep2site) + " " + csv.getInteger(cpep1site);;
                    }
                    //psmID = PSMIDs.toIntValue(key);
                    psmID = key;
                }
            }else
                //psmID=csv.getInteger(cpsmID);
                psmID=csv.getValue(cpsmID);
            
            
            // if we have a column for the peptide length take that value
            // otherwise count all capital letters in the sequence and define <commit_msg>bugfix for reading in csv-files that contain peptide sequences with preceding and succeeding amino-acids with modifications<commit_after>/*
 * Copyright 2015 <NAME> <<EMAIL> at <EMAIL>>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.rappsilber.fdr;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.sql.SQLException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.rappsilber.data.csv.CsvParser;
import org.rappsilber.fdr.entities.PSM;
import org.rappsilber.utils.AutoIncrementValueMap;
import static org.rappsilber.utils.StringUtils.sixDigits;
import org.rappsilber.utils.UpdatableChar;

/**
 *
 * @author lfischer
 */
public class CSVinFDR extends OfflineFDR {
    private String m_source = null;
    private ArrayList<String[]> commandLineColumnMapping;
    private Character delimiter;
    private Character quote;
    private String[][] defaultcolumnmapping=new String[][]{
        new String[]{"peptide1","Peptide1"},
        new String[]{"peptide2","Peptide2"},
        new String[]{"peptide length 1","LengthPeptide1"},
        new String[]{"peptide length 2","LengthPeptide2"},
        new String[]{"peptide link 1","Link1"},
        new String[]{"peptide link 2","Link2"},
        new String[]{"is decoy 1","Protein1decoy"},
        new String[]{"is decoy 2","Protein2decoy"},
        new String[]{"precursor charge","PrecoursorCharge"},
        new String[]{"score","match score"},
        new String[]{"accession1","Protein1"},
        new String[]{"accession2","Protein2"},
        new String[]{"description1","Fasta1"},
        new String[]{"description2","Fasta2"},
        new String[]{"peptide position 1","Start1"},
        new String[]{"peptide position 2","Start2"},
        new String[]{"peptide1 score","Pep1Score"},
        new String[]{"peptide2 score","Pep2Score"},
        new String[]{"experimental mz","expMZ"},
        new String[]{"calculated mass","calcMass"},
        new String[]{"info","info"},
    };
    
    public CSVinFDR() {
    }

    public CSVinFDR(int[] peptideLengthGroups) {
        super(peptideLengthGroups);
    }
    

    
    @Override
    protected ArrayList<String> getPSMHeader() {
        ArrayList<String> ret = super.getPSMHeader();
//        String header = "PSMID" + seperator + "run" + seperator + "scan" 
//                + seperator + "exp charge"  + seperator + "exp m/z" + seperator + "exp mass" + seperator + "exp fractionalmass" 
//                + seperator + "match charge" + seperator + "match mass" + seperator + "match fractionalmass"   
//                
//                + seperator + "Protein1" + seperator + "Description1" + seperator + "Decoy1" + seperator + "Protein2" + seperator + "Description2" + seperator + "Decoy2" + seperator + "Peptide1" + seperator + "Peptide2" + seperator + "PeptidePosition1" + seperator + "PeptidePosition2" + seperator + "PeptideLength1" + seperator + "PeptideLength2" + seperator + "fromSite" + seperator + "ToSite" + seperator + "Charge" + seperator + "Score" + seperator + "isDecoy" + seperator + "isTT" + seperator + "isTD" + seperator + "isDD" + seperator + "fdrGroup" + seperator + "fdr" + seperator + "" + seperator + "PeptidePairFDR" + seperator + "Protein1FDR" + seperator + "Protein2FDR" + seperator + "LinkFDR" + seperator + "PPIFDR" + seperator + seperator + "peptide pair id" + seperator + "link id" + seperator + "ppi id";
        ret.add(2, "exp charge");
        ret.add(3, "exp m/z");
        ret.add(4, "exp mass");
        ret.add(5, "exp fractionalmass");
        ret.add(6, "match charge");
        ret.add(7,  "match mass");
        ret.add(8, "match fractionalmass");
        return ret;
    }

    
    protected ArrayList<String> getPSMOutputLine(PSM pp) {
        ArrayList<String> ret = super.getPSMOutputLine(pp);

        double mz = pp.getExperimentalMZ();
        int charge = pp.getCharge();
        double mass = (mz-1.00727646677)*charge;
        double fraction = mass-Math.floor(mass);
        double calcfraction = pp.getCalcMass()-Math.floor(pp.getCalcMass());

        
        ret.add(2,""+ charge);
        ret.add(3, sixDigits.format(mz));
        ret.add(4, sixDigits.format(mass));
        ret.add(5, sixDigits.format(fraction));
        ret.add(6,""+ pp.getMatchedCharge());
        ret.add(7,  sixDigits.format(pp.getCalcMass()));
        ret.add(8, sixDigits.format(calcfraction));
        

        return ret;
    }    
    

    public void readCSV(File f) throws FileNotFoundException, IOException, ParseException  {
        readCSV(CsvParser.guessCsv(f, 50));
    }

    public void readCSV(CsvParser csv) throws FileNotFoundException, IOException, ParseException  {
        if (csv.getInputFile()!= null)
            m_source = csv.getInputFile().getAbsolutePath();
            

        Integer crun = csv.getColumn("run");
        Integer cscan = csv.getColumn("scan");
        Integer cpsmID = csv.getColumn("psmid");
        int cpep1 = csv.getColumn("peptide1");
        int cpep2 = csv.getColumn("peptide2");
        Integer cpep1len = csv.getColumn("peptide length 1");
        Integer cpep2len = csv.getColumn("peptide length 2");
        int cpep1site = csv.getColumn("peptide link 1");
        int cpep2site = csv.getColumn("peptide link 2");
        int cpep1decoy = csv.getColumn("is decoy 1");
        Integer cpep2decoy = csv.getColumn("is decoy 2");
        int cprecZ = csv.getColumn("precursor charge");
        int cscore = csv.getColumn("score");
        int caccession1 = csv.getColumn("accession1");
        int caccession2 = csv.getColumn("accession2");
        Integer cdescription1 = csv.getColumn("description1");
        Integer cdescription2 = csv.getColumn("description2");
        Integer cpeptide_position1 = csv.getColumn("peptide position 1");
        Integer cpeptide_position2 = csv.getColumn("peptide position 2");
        Integer cscoreratio = csv.getColumn("score ratio");
        Integer cPepScore1 = csv.getColumn("peptide1 score");
        Integer cPepScore2 = csv.getColumn("peptide2 score");
        Integer cCrosslinker = csv.getColumn("crosslinker");
        Integer cExpMZ = csv.getColumn("experimental mz");
        Integer cCalcMass = csv.getColumn("calculated mass");
        Integer cInfo = csv.getColumn("info");
        
        int noID = 0;
        AutoIncrementValueMap<String> PSMIDs = new AutoIncrementValueMap<String>();
        int lineNumber =0;
        
        int noPSMID =0;
        while (csv.next()) {
            lineNumber++;
            String psmID;

            String pepSeq1 = csv.getValue(cpep1);
            String pepSeq2 = csv.getValue(cpep2);
            String terminalAminoAcids = "([A-Z\\--][a-z]*|(\\[[A-Z\\--][a-z]*\\])+)";

            // if the sequence looks like K.PEPTIDEK.A assume that the first and 
            // last aminoacid are leading and trailing aminoacids and not really part of the peptide
            if (pepSeq1.matches(terminalAminoAcids+"\\..*"+terminalAminoAcids)) {
                pepSeq1 = pepSeq1.replaceAll("^" + terminalAminoAcids +"\\.", "").replaceAll("\\."+terminalAminoAcids +"$", "");
            }
            if (pepSeq2.matches(terminalAminoAcids+"\\..*"+terminalAminoAcids)) {
                pepSeq2 = pepSeq2.replaceAll("^" + terminalAminoAcids +"\\.", "").replaceAll("\\."+terminalAminoAcids +"$", "");
            }

            
            Integer site1 = csv.getInteger(cpep1site,-1);
            Integer site2 = csv.getInteger(cpep2site,-1); //pepSeq2 == null || pepSeq2.trim().isEmpty() ? -1 : csv.getInteger(cpep2site,-1);
            
            // do we have to generate an ID?
            if (cpsmID == null) {
                if (cscan == null || crun == null) {
                    psmID = Integer.toString(noPSMID++);
                } else {
                    String key = "Scan: " + csv.getValue(cscan) + " Run: " + csv.getValue(crun);
                    int c= pepSeq1.compareTo(pepSeq2) ;
                    if (c > 0 || (c==0 && site1 > site2) ) {
                        key=key +" P1_" + csv.getValue(cpep1) + " P2_" + csv.getValue(cpep2) + " " + csv.getInteger(cpep1site) + " " + csv.getInteger(cpep2site);
                    } else {
                        key=key +" P1_" + csv.getValue(cpep2) + " P2_" + csv.getValue(cpep1) + " " + csv.getInteger(cpep2site) + " " + csv.getInteger(cpep1site);;
                    }
                    //psmID = PSMIDs.toIntValue(key);
                    psmID = key;
                }
            }else
                //psmID=csv.getInteger(cpsmID);
                psmID=csv.getValue(cpsmID);
            
            
            // if we have a column for the peptide length take that value
            // otherwise count all capital letters in the sequence and define 
{'fraction_non_alphanumeric': 0.13980121002592913, 'fraction_numerical': 0.016853932584269662, 'mean_word_length': 2.5588235294117645, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4884003', 'n_tokens_mistral': 6106, 'n_tokens_neox': 5690, 'n_words': 1300}