<commit_before>package org.gbif.occurrence.processor.interpreting;

import org.gbif.api.model.occurrence.Occurrence;
import org.gbif.api.model.occurrence.VerbatimOccurrence;
import org.gbif.api.vocabulary.BasisOfRecord;
import org.gbif.api.vocabulary.EstablishmentMeans;
import org.gbif.api.vocabulary.LifeStage;
import org.gbif.api.vocabulary.OccurrenceIssue;
import org.gbif.api.vocabulary.Sex;
import org.gbif.api.vocabulary.TypeStatus;
import org.gbif.common.parsers.BasisOfRecordParser;
import org.gbif.common.parsers.EstablishmentMeansParser;
import org.gbif.common.parsers.LifeStageParser;
import org.gbif.common.parsers.NumberParser;
import org.gbif.common.parsers.SexParser;
import org.gbif.common.parsers.TypeStatusParser;
import org.gbif.common.parsers.TypifiedNameParser;
import org.gbif.common.parsers.core.Parsable;
import org.gbif.common.parsers.core.ParseResult;
import org.gbif.common.parsers.UrlParser;
import org.gbif.dwc.terms.DcTerm;
import org.gbif.dwc.terms.DwcTerm;
import org.gbif.dwc.terms.GbifTerm;
import org.gbif.occurrence.processor.interpreting.result.OccurrenceInterpretationResult;

import java.util.Date;
import java.util.List;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;



/**
 * Interprets/Validates verbatim occurrence records.
 * This class doesn't persist any information, it only collects possible issues and generates a interpreted version
 * of the verbatim record.
 */
@Singleton
public class OccurrenceInterpreter {

  @FunctionalInterface
  private interface Interpreter{
    void interpret(VerbatimOccurrence verbatim, Occurrence occurrence);
  }

  private static final Logger LOG = LoggerFactory.getLogger(VerbatimOccurrenceInterpreter.class);

  private static final TypeStatusParser TYPE_PARSER = TypeStatusParser.getInstance();
  private static final Parsable<String> TYPE_NAME_PARSER = TypifiedNameParser.getInstance();
  private static final BasisOfRecordParser BOR_PARSER = BasisOfRecordParser.getInstance();
  private static final SexParser SEX_PARSER = SexParser.getInstance();
  private static final EstablishmentMeansParser EST_PARSER = EstablishmentMeansParser.getInstance();
  private static final LifeStageParser LST_PARSER = LifeStageParser.getInstance();

  private final DatasetInfoInterpreter datasetInfoInterpreter;


  //Holds the list of Interpreters that will be applied
  private final List<Interpreter> interpreters;

  @Inject
  public OccurrenceInterpreter(DatasetInfoInterpreter datasetInfoInterpreter,
                               TaxonomyInterpreter taxonomyInterpreter, LocationInterpreter locationInterpreter) {
    this.datasetInfoInterpreter = datasetInfoInterpreter;
    //the list of interpreters is initialized with all the interpretations methods used
    interpreters = new ImmutableList.Builder<Interpreter>().add(locationInterpreter::interpretLocation,
                                     taxonomyInterpreter::interpretTaxonomy,
                                     MultiMediaInterpreter::interpretMedia,
                                     this::interpretDatasetInfo,
                                     OccurrenceInterpreter::interpretBor,
                                     OccurrenceInterpreter::interpretSex,
                                     OccurrenceInterpreter::interpretEstablishmentMeans,
                                     OccurrenceInterpreter::interpretLifeStage,
                                     OccurrenceInterpreter::interpretTypification,
                                     TemporalInterpreter::interpretTemporal,
                                     OccurrenceInterpreter::interpretReferences,
                                     OccurrenceInterpreter::interpretIndividualCount).build();
  }

  /**
   * Interpret all the verbatim fields into our standard Occurrence fields.
   * TODO: send messages/write logs for interpretation errors
   *
   * @param verbatim the verbatim occurrence to interpret
   *
   * @return an OccurrenceInterpretationResult that contains an "updated" Occurrence with interpreted fields and an
   * "original"
   * occurrence iff this was an update to an existing record (will be null otherwise)
   */
  public OccurrenceInterpretationResult interpret(VerbatimOccurrence verbatim) {
    Occurrence occ = new Occurrence(verbatim);
    interpreters.stream().forEach(interpreter ->  {
      try {
        interpreter.interpret(verbatim,occ);
      } catch (Exception e) {
        LOG.warn("Caught a runtime exception during interpretation", e);
        occ.addIssue(OccurrenceIssue.INTERPRETATION_ERROR);
      }
    });
    occ.setLastInterpreted(new Date());
    return new OccurrenceInterpretationResult(null, occ);
  }

  /**
   *  This method was created to be follow the Interpreter functional interface contract.
   */
  private void interpretDatasetInfo(VerbatimOccurrence verbatim, Occurrence occ) {
    datasetInfoInterpreter.interpretDatasetInfo(occ);
  }

  private static void interpretReferences(VerbatimOccurrence verbatim, Occurrence occ) {
    if (verbatim.hasVerbatimField(DcTerm.references)) {
      String val = verbatim.getVerbatimField(DcTerm.references);
      if (!Strings.isNullOrEmpty(val)) {
        occ.setReferences(UrlParser.parse(val));
        if (occ.getReferences() == null) {
          occ.getIssues().add(OccurrenceIssue.REFERENCES_URI_INVALID);
        }
      }
    }
  }<commit_msg>adding the verbatim record to the original record in the interpretation result<commit_after>package org.gbif.occurrence.processor.interpreting;

import org.gbif.api.model.occurrence.Occurrence;
import org.gbif.api.model.occurrence.VerbatimOccurrence;
import org.gbif.api.vocabulary.BasisOfRecord;
import org.gbif.api.vocabulary.EstablishmentMeans;
import org.gbif.api.vocabulary.LifeStage;
import org.gbif.api.vocabulary.OccurrenceIssue;
import org.gbif.api.vocabulary.Sex;
import org.gbif.api.vocabulary.TypeStatus;
import org.gbif.common.parsers.BasisOfRecordParser;
import org.gbif.common.parsers.EstablishmentMeansParser;
import org.gbif.common.parsers.LifeStageParser;
import org.gbif.common.parsers.NumberParser;
import org.gbif.common.parsers.SexParser;
import org.gbif.common.parsers.TypeStatusParser;
import org.gbif.common.parsers.TypifiedNameParser;
import org.gbif.common.parsers.core.Parsable;
import org.gbif.common.parsers.core.ParseResult;
import org.gbif.common.parsers.UrlParser;
import org.gbif.dwc.terms.DcTerm;
import org.gbif.dwc.terms.DwcTerm;
import org.gbif.dwc.terms.GbifTerm;
import org.gbif.occurrence.processor.interpreting.result.OccurrenceInterpretationResult;

import java.util.Date;
import java.util.List;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;



/**
 * Interprets/Validates verbatim occurrence records.
 * This class doesn't persist any information, it only collects possible issues and generates a interpreted version
 * of the verbatim record.
 */
@Singleton
public class OccurrenceInterpreter {

  @FunctionalInterface
  private interface Interpreter{
    void interpret(VerbatimOccurrence verbatim, Occurrence occurrence);
  }

  private static final Logger LOG = LoggerFactory.getLogger(VerbatimOccurrenceInterpreter.class);

  private static final TypeStatusParser TYPE_PARSER = TypeStatusParser.getInstance();
  private static final Parsable<String> TYPE_NAME_PARSER = TypifiedNameParser.getInstance();
  private static final BasisOfRecordParser BOR_PARSER = BasisOfRecordParser.getInstance();
  private static final SexParser SEX_PARSER = SexParser.getInstance();
  private static final EstablishmentMeansParser EST_PARSER = EstablishmentMeansParser.getInstance();
  private static final LifeStageParser LST_PARSER = LifeStageParser.getInstance();

  private final DatasetInfoInterpreter datasetInfoInterpreter;


  //Holds the list of Interpreters that will be applied
  private final List<Interpreter> interpreters;

  @Inject
  public OccurrenceInterpreter(DatasetInfoInterpreter datasetInfoInterpreter,
                               TaxonomyInterpreter taxonomyInterpreter, LocationInterpreter locationInterpreter) {
    this.datasetInfoInterpreter = datasetInfoInterpreter;
    //the list of interpreters is initialized with all the interpretations methods used
    interpreters = new ImmutableList.Builder<Interpreter>().add(locationInterpreter::interpretLocation,
                                     taxonomyInterpreter::interpretTaxonomy,
                                     MultiMediaInterpreter::interpretMedia,
                                     this::interpretDatasetInfo,
                                     OccurrenceInterpreter::interpretBor,
                                     OccurrenceInterpreter::interpretSex,
                                     OccurrenceInterpreter::interpretEstablishmentMeans,
                                     OccurrenceInterpreter::interpretLifeStage,
                                     OccurrenceInterpreter::interpretTypification,
                                     TemporalInterpreter::interpretTemporal,
                                     OccurrenceInterpreter::interpretReferences,
                                     OccurrenceInterpreter::interpretIndividualCount).build();
  }

  /**
   * Interpret all the verbatim fields into our standard Occurrence fields.
   * TODO: send messages/write logs for interpretation errors
   *
   * @param verbatim the verbatim occurrence to interpret
   *
   * @return an OccurrenceInterpretationResult that contains an "updated" Occurrence with interpreted fields and an
   * "original"
   * occurrence iff this was an update to an existing record (will be null otherwise)
   */
  public OccurrenceInterpretationResult interpret(VerbatimOccurrence verbatim) {
    Occurrence occ = new Occurrence(verbatim);
    interpreters.stream().forEach(interpreter ->  {
      try {
        interpreter.interpret(verbatim,occ);
      } catch (Exception e) {
        LOG.warn("Caught a runtime exception during interpretation", e);
        occ.addIssue(OccurrenceIssue.INTERPRETATION_ERROR);
      }
    });
    occ.setLastInterpreted(new Date());
    return new OccurrenceInterpretationResult(new Occurrence(verbatim), occ);
  }

  /**
   *  This method was created to be follow the Interpreter functional interface contract.
   */
  private void interpretDatasetInfo(VerbatimOccurrence verbatim, Occurrence occ) {
    datasetInfoInterpreter.interpretDatasetInfo(occ);
  }

  private static void interpretReferences(VerbatimOccurrence verbatim, Occurrence occ) {
    if (verbatim.hasVerbatimField(DcTerm.references)) {
      String val = verbatim.getVerbatimField(DcTerm.references);
      if (!Strings.isNullOrEmpty(val)) {
        occ.setReferences(UrlParser.parse(val));
        if (occ.getReferences() == null) {
          occ.getIssues().add(OccurrenceIssue.REFERENCES_URI_INVALID);
        }
      }
    }
  }
{'fraction_non_alphanumeric': 0.07593803170054625, 'fraction_numerical': 0.00035819826273842574, 'mean_word_length': 3.5714285714285716, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6005416', 'n_tokens_mistral': 3131, 'n_tokens_neox': 3047, 'n_words': 636}