<commit_before>/*
 * Copyright 2020 Google LLC, MobilityData IO
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.mobilitydata.gtfsvalidator.parsing;

import com.google.i18n.phonenumbers.PhoneNumberUtil;
import java.math.BigDecimal;
import java.time.ZoneId;
import java.time.zone.ZoneRulesException;
import java.util.Currency;
import java.util.Locale;
import java.util.function.Function;
import java.util.function.Predicate;
import javax.annotation.Nullable;
import org.apache.commons.validator.routines.EmailValidator;
import org.apache.commons.validator.routines.UrlValidator;
import org.mobilitydata.gtfsvalidator.input.GtfsFeedName;
import org.mobilitydata.gtfsvalidator.notice.EmptyRowNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidColorNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidCurrencyNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidDateNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidEmailNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidFloatNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidIntegerNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidLanguageCodeNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidPhoneNumberNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidRowLengthError;
import org.mobilitydata.gtfsvalidator.notice.InvalidTimeNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidTimezoneNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidUrlNotice;
import org.mobilitydata.gtfsvalidator.notice.LeadingOrTrailingWhitespacesNotice;
import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFieldError;
import org.mobilitydata.gtfsvalidator.notice.NewLineInValueNotice;
import org.mobilitydata.gtfsvalidator.notice.NonAsciiOrNonPrintableCharNotice;
import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
import org.mobilitydata.gtfsvalidator.notice.NumberOutOfRangeError;
import org.mobilitydata.gtfsvalidator.notice.SeverityLevel;
import org.mobilitydata.gtfsvalidator.notice.UnexpectedEnumValueError;
import org.mobilitydata.gtfsvalidator.notice.ValidationNotice;
import org.mobilitydata.gtfsvalidator.type.GtfsColor;
import org.mobilitydata.gtfsvalidator.type.GtfsDate;
import org.mobilitydata.gtfsvalidator.type.GtfsTime;

/**
 * Parses cells of a CSV row as values of requested data types.
 *
 * <p>Interface functions of this class receive an instance of {@code NoticeContainer}. If a cell
 * value cannot be parsed, these functions add a notice to the container, return null and don't
 * throw an exception.
 */
public class RowParser {

  public static final boolean REQUIRED = true;
  public static final boolean OPTIONAL = false;
  private final NoticeContainer noticeContainer;
  private final GtfsFeedName feedName;
  private CsvRow row;
  private boolean parseErrorsInRow;

  public RowParser(GtfsFeedName feedName, NoticeContainer noticeContainer) {
    this.feedName = feedName;
    this.noticeContainer = noticeContainer;
  }

  public NoticeContainer getNoticeContainer() {
    return noticeContainer;
  }

  public void setRow(CsvRow row) {
    this.row = row;
    this.parseErrorsInRow = false;
  }

  public boolean hasParseErrorsInRow() {
    return parseErrorsInRow;
  }

  /**
   * Checks whether the row lengths (cell count) is the same as the amount of file headers.
   *
   * <p>This function may add notices to {@code noticeContainer}.
   *
   * @return true if the row length is equal to column count
   */
  public boolean checkRowLength() {
    if (row.getColumnCount() == 0) {
      // Empty row.
      return false;
    }

    CsvFile csvFile = row.getCsvFile();
    if (row.getColumnCount() == 1 && row.asString(0) == null) {
      // If the last row has only spaces and does not end with a newline, then Univocity parser
      // interprets it as a non-empty row that has a single column which is empty (sic!). We are
      // unsure if this is a bug or feature in Univocity, so we show a warning.
      addNoticeInRow(new EmptyRowNotice(csvFile.getFileName(), row.getRowNumber()));
      return false;
    }

    if (row.getColumnCount() != csvFile.getColumnCount()) {
      addNoticeInRow(
          new InvalidRowLengthError(
              csvFile.getFileName(),
              row.getRowNumber(),
              row.getColumnCount(),
              csvFile.getColumnCount()));
      return false;
    }
    return true;
  }

  @Nullable
  public String asString(int columnIndex, boolean required) {
    String s = row.asString(columnIndex);
    if (required && s == null) {
      addNoticeInRow(
          new MissingRequiredFieldError(
              row.getFileName(), row.getRowNumber(), row.getColumnName(columnIndex)));
    }
    if (s != null) {
      if (s.indexOf('\n') != -1 || s.indexOf('\r') != -1) {
        addNoticeInRow(
            new NewLineInValueNotice(
                row.getFileName(), row.getRowNumber(), row.getColumnName(columnIndex), s));
      }
      final String stripped = s.strip();
      if (stripped.length() < s.length()) {
        addNoticeInRow(
            new LeadingOrTrailingWhitespacesNotice(
                row.getFileName(), row.getRowNumber(), row.getColumnName(columnIndex), s));
        s = stripped;<commit_msg>fix: Use String.trim instead of String.strip (#758)<commit_after>/*
 * Copyright 2020 Google LLC, MobilityData IO
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.mobilitydata.gtfsvalidator.parsing;

import com.google.i18n.phonenumbers.PhoneNumberUtil;
import java.math.BigDecimal;
import java.time.ZoneId;
import java.time.zone.ZoneRulesException;
import java.util.Currency;
import java.util.Locale;
import java.util.function.Function;
import java.util.function.Predicate;
import javax.annotation.Nullable;
import org.apache.commons.validator.routines.EmailValidator;
import org.apache.commons.validator.routines.UrlValidator;
import org.mobilitydata.gtfsvalidator.input.GtfsFeedName;
import org.mobilitydata.gtfsvalidator.notice.EmptyRowNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidColorNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidCurrencyNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidDateNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidEmailNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidFloatNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidIntegerNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidLanguageCodeNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidPhoneNumberNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidRowLengthError;
import org.mobilitydata.gtfsvalidator.notice.InvalidTimeNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidTimezoneNotice;
import org.mobilitydata.gtfsvalidator.notice.InvalidUrlNotice;
import org.mobilitydata.gtfsvalidator.notice.LeadingOrTrailingWhitespacesNotice;
import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFieldError;
import org.mobilitydata.gtfsvalidator.notice.NewLineInValueNotice;
import org.mobilitydata.gtfsvalidator.notice.NonAsciiOrNonPrintableCharNotice;
import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
import org.mobilitydata.gtfsvalidator.notice.NumberOutOfRangeError;
import org.mobilitydata.gtfsvalidator.notice.SeverityLevel;
import org.mobilitydata.gtfsvalidator.notice.UnexpectedEnumValueError;
import org.mobilitydata.gtfsvalidator.notice.ValidationNotice;
import org.mobilitydata.gtfsvalidator.type.GtfsColor;
import org.mobilitydata.gtfsvalidator.type.GtfsDate;
import org.mobilitydata.gtfsvalidator.type.GtfsTime;

/**
 * Parses cells of a CSV row as values of requested data types.
 *
 * <p>Interface functions of this class receive an instance of {@code NoticeContainer}. If a cell
 * value cannot be parsed, these functions add a notice to the container, return null and don't
 * throw an exception.
 */
public class RowParser {

  public static final boolean REQUIRED = true;
  public static final boolean OPTIONAL = false;
  private final NoticeContainer noticeContainer;
  private final GtfsFeedName feedName;
  private CsvRow row;
  private boolean parseErrorsInRow;

  public RowParser(GtfsFeedName feedName, NoticeContainer noticeContainer) {
    this.feedName = feedName;
    this.noticeContainer = noticeContainer;
  }

  public NoticeContainer getNoticeContainer() {
    return noticeContainer;
  }

  public void setRow(CsvRow row) {
    this.row = row;
    this.parseErrorsInRow = false;
  }

  public boolean hasParseErrorsInRow() {
    return parseErrorsInRow;
  }

  /**
   * Checks whether the row lengths (cell count) is the same as the amount of file headers.
   *
   * <p>This function may add notices to {@code noticeContainer}.
   *
   * @return true if the row length is equal to column count
   */
  public boolean checkRowLength() {
    if (row.getColumnCount() == 0) {
      // Empty row.
      return false;
    }

    CsvFile csvFile = row.getCsvFile();
    if (row.getColumnCount() == 1 && row.asString(0) == null) {
      // If the last row has only spaces and does not end with a newline, then Univocity parser
      // interprets it as a non-empty row that has a single column which is empty (sic!). We are
      // unsure if this is a bug or feature in Univocity, so we show a warning.
      addNoticeInRow(new EmptyRowNotice(csvFile.getFileName(), row.getRowNumber()));
      return false;
    }

    if (row.getColumnCount() != csvFile.getColumnCount()) {
      addNoticeInRow(
          new InvalidRowLengthError(
              csvFile.getFileName(),
              row.getRowNumber(),
              row.getColumnCount(),
              csvFile.getColumnCount()));
      return false;
    }
    return true;
  }

  @Nullable
  public String asString(int columnIndex, boolean required) {
    String s = row.asString(columnIndex);
    if (required && s == null) {
      addNoticeInRow(
          new MissingRequiredFieldError(
              row.getFileName(), row.getRowNumber(), row.getColumnName(columnIndex)));
    }
    if (s != null) {
      if (s.indexOf('\n') != -1 || s.indexOf('\r') != -1) {
        addNoticeInRow(
            new NewLineInValueNotice(
                row.getFileName(), row.getRowNumber(), row.getColumnName(columnIndex), s));
      }
      final String trimmed = s.trim();
      if (trimmed.length() < s.length()) {
        addNoticeInRow(
            new LeadingOrTrailingWhitespacesNotice(
                row.getFileName(), row.getRowNumber(), row.getColumnName(columnIndex), s));
        s = trimmed;
{'fraction_non_alphanumeric': 0.08985657508208053, 'fraction_numerical': 0.002851218247796786, 'mean_word_length': 4.078982009653357, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6001717', 'n_tokens_mistral': 3369, 'n_tokens_neox': 3295, 'n_words': 875}