<commit_before>/* Copyright (C) 2019 <NAME>
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  You may obtain a copy of the License at
 *
 *     https://www.gnu.org/licenses/agpl-3.0.txt
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 */
package bzh.plealog.bioinfo.io.searchresult.ncbi;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.util.List;

import javax.xml.bind.JAXB;

import bzh.plealog.bioinfo.api.data.feature.Feature;
import bzh.plealog.bioinfo.api.data.feature.FeatureTable;
import bzh.plealog.bioinfo.api.data.searchresult.SRHit;
import bzh.plealog.bioinfo.api.data.searchresult.SRHsp;
import bzh.plealog.bioinfo.api.data.searchresult.SRHspSequence;
import bzh.plealog.bioinfo.api.data.searchresult.SRIteration;
import bzh.plealog.bioinfo.api.data.searchresult.SROutput;
import bzh.plealog.bioinfo.api.data.searchresult.SRParameters;
import bzh.plealog.bioinfo.api.data.searchresult.SRRequestInfo;
import bzh.plealog.bioinfo.api.data.searchresult.io.SRLoader;
import bzh.plealog.bioinfo.api.data.searchresult.io.SRLoaderException;
import bzh.plealog.bioinfo.api.data.sequence.BankSequenceInfo;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.BlastOutput2;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.BlastXML2;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Hit;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.HitDescr;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Hsp;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Iteration;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Parameters;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Report;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Report.ReportParams;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Results;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Search;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Search.SearchHits;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Search.SearchStat;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Statistics;
import bzh.plealog.bioinfo.data.feature.IFeatureTable;
import bzh.plealog.bioinfo.data.searchresult.ISRHit;
import bzh.plealog.bioinfo.data.searchresult.ISRHsp;
import bzh.plealog.bioinfo.data.searchresult.ISRHspScore;
import bzh.plealog.bioinfo.data.searchresult.ISRHspSequence;
import bzh.plealog.bioinfo.data.searchresult.ISRIteration;
import bzh.plealog.bioinfo.data.searchresult.ISROutput;
import bzh.plealog.bioinfo.data.searchresult.ISRParameters;
import bzh.plealog.bioinfo.data.searchresult.ISRRequestInfo;
import bzh.plealog.bioinfo.data.searchresult.ISRStatistics;
import bzh.plealog.bioinfo.data.sequence.IBankSequenceInfo;
import bzh.plealog.bioinfo.util.DAlphabetUtils;
import bzh.plealog.bioinfo.util.ZipUtil;

public class BlastLoader2 implements SRLoader {
  public static final String SYSTEM_NAME = "XMLBlast2";
  public static String AA_TYPE = "aa";
  public static String NUCL_TYPE = "nucleotide";
  public static String SOURCE_FEAT_TYPE = "source";
  public static String DBXREF_QUAL_TYPE = "dbxref";
  public static String TAXON_CODING_TYPE = "taxon; ";

  private static final String BL2SEQ_ERROR = "Does not handle Blast2Seq results";
  
  public BlastLoader2() {
    
  }

  @Override
  public String getSystemName() {
    return SYSTEM_NAME;
  }

  @Override
  public String getName() {
    return "NCBI XML2 single file formatted data";
  }

  @Override
  public String getVersion() {
    return "1.0";
  }

  @Override
  public boolean canRead(File f) {
    File            f2;
    String          line;
    boolean         bOk = false;
    boolean         isZip = false;
    int             i = 0;

    try (FileInputStream fis = new FileInputStream(f)) {
      f2 = ZipUtil.extract(fis, f.getParent());
      isZip = true;
    }
    catch(Exception e){
      f2 = f;
    }
    try (BufferedReader bReader= new BufferedReader(new FileReader(f2))){ 
      while ((line=bReader.readLine()) != null) {
        if (line.indexOf("<BlastXML2")>=0){
          bOk=true;
        }
        if (line.indexOf("<BlastOutput2>")>=0){
          bOk&=true;
          break;
        }
        i++;
        //above elements have to be located in file head
        if (i>20)
          break;
      }
    } 
    catch (Exception e) {  
    } 
    if (isZip)
      try{f2.delete();}catch(Exception ex){}

    return bOk;
  }

  private ISRHsp initHsp(Hsp hsp, boolean isProt, boolean isBlastn){
    ISRHsp         iHsp;
    ISRHspScore    score;
    ISRHspSequence query, hit, midline;
    String        seq;
    int           from, to;
    boolean       doRCalignment = false;

    iHsp = new ISRHsp();
    score = new ISRHspScore();
    query = new ISRHspSequence();
    hit = new ISRHspSequence();
    midline = new ISRHspSequence();

    //scores
    score.setBitScore(hsp.getHspBitScore());
    score.setScore(hsp.getHspScore());
    score.setEvalue(hsp.getHspEvalue());
    score.setIdentity(hsp.getHspIdentity());
    score.setPositive(hsp.getHspPositive());
    score.setGaps(hsp.getHspGaps());
    score.setAlignLen(hsp.getHspAlignLen());
    if (hsp.getHspDensity()!=null)
      score.setDensity(hsp.getHspDensity());
<commit_msg>fix issues when BlastResult2 has missing values<commit_after>/* Copyright (C) 2019 <NAME>
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  You may obtain a copy of the License at
 *
 *     https://www.gnu.org/licenses/agpl-3.0.txt
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 */
package bzh.plealog.bioinfo.io.searchresult.ncbi;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.util.List;

import javax.xml.bind.JAXB;

import bzh.plealog.bioinfo.api.data.feature.Feature;
import bzh.plealog.bioinfo.api.data.feature.FeatureTable;
import bzh.plealog.bioinfo.api.data.searchresult.SRHit;
import bzh.plealog.bioinfo.api.data.searchresult.SRHsp;
import bzh.plealog.bioinfo.api.data.searchresult.SRHspSequence;
import bzh.plealog.bioinfo.api.data.searchresult.SRIteration;
import bzh.plealog.bioinfo.api.data.searchresult.SROutput;
import bzh.plealog.bioinfo.api.data.searchresult.SRParameters;
import bzh.plealog.bioinfo.api.data.searchresult.SRRequestInfo;
import bzh.plealog.bioinfo.api.data.searchresult.io.SRLoader;
import bzh.plealog.bioinfo.api.data.searchresult.io.SRLoaderException;
import bzh.plealog.bioinfo.api.data.sequence.BankSequenceInfo;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.BlastOutput2;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.BlastXML2;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Hit;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.HitDescr;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Hsp;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Iteration;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Parameters;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Report;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Report.ReportParams;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Results;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Search;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Search.SearchHits;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Search.SearchStat;
import bzh.plealog.bioinfo.data.blast.loader.ncbi2.Statistics;
import bzh.plealog.bioinfo.data.feature.IFeatureTable;
import bzh.plealog.bioinfo.data.searchresult.ISRHit;
import bzh.plealog.bioinfo.data.searchresult.ISRHsp;
import bzh.plealog.bioinfo.data.searchresult.ISRHspScore;
import bzh.plealog.bioinfo.data.searchresult.ISRHspSequence;
import bzh.plealog.bioinfo.data.searchresult.ISRIteration;
import bzh.plealog.bioinfo.data.searchresult.ISROutput;
import bzh.plealog.bioinfo.data.searchresult.ISRParameters;
import bzh.plealog.bioinfo.data.searchresult.ISRRequestInfo;
import bzh.plealog.bioinfo.data.searchresult.ISRStatistics;
import bzh.plealog.bioinfo.data.sequence.IBankSequenceInfo;
import bzh.plealog.bioinfo.util.DAlphabetUtils;
import bzh.plealog.bioinfo.util.ZipUtil;

public class BlastLoader2 implements SRLoader {
  public static final String SYSTEM_NAME = "XMLBlast2";
  public static String AA_TYPE = "aa";
  public static String NUCL_TYPE = "nucleotide";
  public static String SOURCE_FEAT_TYPE = "source";
  public static String DBXREF_QUAL_TYPE = "dbxref";
  public static String TAXON_CODING_TYPE = "taxon; ";

  private static final String BL2SEQ_ERROR = "Does not handle Blast2Seq results";
  
  public BlastLoader2() {
    
  }

  @Override
  public String getSystemName() {
    return SYSTEM_NAME;
  }

  @Override
  public String getName() {
    return "NCBI XML2 single file formatted data";
  }

  @Override
  public String getVersion() {
    return "1.0";
  }

  @Override
  public boolean canRead(File f) {
    File            f2;
    String          line;
    boolean         bOk = false;
    boolean         isZip = false;
    int             i = 0;

    try (FileInputStream fis = new FileInputStream(f)) {
      f2 = ZipUtil.extract(fis, f.getParent());
      isZip = true;
    }
    catch(Exception e){
      f2 = f;
    }
    try (BufferedReader bReader= new BufferedReader(new FileReader(f2))){ 
      while ((line=bReader.readLine()) != null) {
        if (line.indexOf("<BlastXML2")>=0){
          bOk=true;
        }
        if (line.indexOf("<BlastOutput2>")>=0){
          bOk&=true;
          break;
        }
        i++;
        //above elements have to be located in file head
        if (i>20)
          break;
      }
    } 
    catch (Exception e) {  
    } 
    if (isZip)
      try{f2.delete();}catch(Exception ex){}

    return bOk;
  }

  private ISRHsp initHsp(Hsp hsp, boolean isProt, boolean isBlastn){
    ISRHsp         iHsp;
    ISRHspScore    score;
    ISRHspSequence query, hit, midline;
    String        seq;
    int           from, to;
    boolean       doRCalignment = false;

    iHsp = new ISRHsp();
    score = new ISRHspScore();
    query = new ISRHspSequence();
    hit = new ISRHspSequence();
    midline = new ISRHspSequence();

    //scores
    score.setBitScore(hsp.getHspBitScore());
    score.setScore(hsp.getHspScore());
    score.setEvalue(hsp.getHspEvalue());
    score.setIdentity(hsp.getHspIdentity());
    if (hsp.getHspPositive()!=null)
      score.setPositive(hsp.getHspPositive());
    score.setGaps(hsp.getHspGaps());
    score.setAlignLen(hsp.getHspAlignLen());
    if (hsp.getHspDensity()!=null)
      score.setDensity(hsp.getHspDensity());

{'fraction_non_alphanumeric': 0.10973248856036606, 'fraction_numerical': 0.007655755015839493, 'mean_word_length': 3.6731085526315788, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7588830', 'n_tokens_mistral': 4090, 'n_tokens_neox': 3910, 'n_words': 766}