<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.search.robotstxt;

import com.google.common.flogger.FluentLogger;
import java.nio.charset.StandardCharsets;
import java.util.logging.Level;

/** Robots.txt parser implementation. */
public class RobotsParser extends Parser {
  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

  public RobotsParser(ParseHandler parseHandler) {
    super(parseHandler);
  }

  private static boolean isWhitespace(final char ch) {
    return ch == ' ' || ch == '\t';
  }

  /**
   * Extracts substring between given indexes and trims preceding and succeeding whitespace
   * characters.
   *
   * @param string string to extract from
   * @param beginIndex the beginning index, inclusive
   * @param endIndex the ending index, exclusive
   * @return extracted substring with trimmed whitespaces
   * @throws ParseException if there are only whitespace characters between given indexes
   */
  private static String trimBounded(final String string, final int beginIndex, final int endIndex)
      throws ParseException {
    int begin = beginIndex;
    int end = endIndex;
    while (begin < endIndex && isWhitespace(string.charAt(begin))) {
      begin++;
    }
    while (end > beginIndex && isWhitespace(string.charAt(end - 1))) {
      end--;
    }
    if (begin >= end) {
      throw new ParseException();
    } else {
      return string.substring(begin, end);
    }
  }

  private static DirectiveType parseDirective(final String key) {
    if (key.equalsIgnoreCase("user-agent")) {
      return DirectiveType.USER_AGENT;
    } else {
      try {
        return DirectiveType.valueOf(key.toUpperCase());
      } catch (IllegalArgumentException e) {
        return DirectiveType.UNKNOWN;
      }
    }
  }

  private static void log(
      final Level level,
      final String message,
      final String robotsTxtBody,
      final int lineBegin,
      final int lineEnd,
      final int lineNumber) {
    logger.at(level).log(
        "%s%nAt line %d:%n%s\t", message, lineNumber, robotsTxtBody.substring(lineBegin, lineEnd));
  }

  private void parseLine(
      final String robotsTxtBody, final int lineBegin, final int lineEnd, final int lineNumber) {
    int limit = lineEnd;
    int separator = lineEnd;
    int whitespaceSeparator = lineEnd;
    boolean hasContents = false;

    for (int i = lineBegin; i < lineEnd; i++) {
      final char ch = robotsTxtBody.charAt(i);
      if (!isWhitespace(ch)) {
        hasContents = true;
      }
      if (isWhitespace(ch) && hasContents) {
        whitespaceSeparator = i;
      }
      if (separator == lineEnd && ch == ':') {
        separator = i;
      }
      if (ch == '#') {
        limit = i;
        break;
      }
    }

    if (separator == lineEnd) {
      // Google-specific optimization: some people forget the colon, so we need to
      // accept whitespace in its stead.
      if (whitespaceSeparator != lineEnd) {
        log(
            Level.INFO,
            "Assuming whitespace as a separator.",
            robotsTxtBody,
            lineBegin,
            lineEnd,
            lineNumber);
        separator = whitespaceSeparator;
      } else {
        if (hasContents) {
          log(Level.WARNING, "No separator found.", robotsTxtBody, lineBegin, lineEnd, lineNumber);
        }
        return;
      }
    }

    final String key;<commit_msg>Fixed missing contents logging.<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.search.robotstxt;

import com.google.common.flogger.FluentLogger;
import java.nio.charset.StandardCharsets;
import java.util.logging.Level;

/** Robots.txt parser implementation. */
public class RobotsParser extends Parser {
  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

  public RobotsParser(ParseHandler parseHandler) {
    super(parseHandler);
  }

  private static boolean isWhitespace(final char ch) {
    return ch == ' ' || ch == '\t';
  }

  /**
   * Extracts substring between given indexes and trims preceding and succeeding whitespace
   * characters.
   *
   * @param string string to extract from
   * @param beginIndex the beginning index, inclusive
   * @param endIndex the ending index, exclusive
   * @return extracted substring with trimmed whitespaces
   * @throws ParseException if there are only whitespace characters between given indexes
   */
  private static String trimBounded(final String string, final int beginIndex, final int endIndex)
      throws ParseException {
    int begin = beginIndex;
    int end = endIndex;
    while (begin < endIndex && isWhitespace(string.charAt(begin))) {
      begin++;
    }
    while (end > beginIndex && isWhitespace(string.charAt(end - 1))) {
      end--;
    }
    if (begin >= end) {
      throw new ParseException();
    } else {
      return string.substring(begin, end);
    }
  }

  private static DirectiveType parseDirective(final String key) {
    if (key.equalsIgnoreCase("user-agent")) {
      return DirectiveType.USER_AGENT;
    } else {
      try {
        return DirectiveType.valueOf(key.toUpperCase());
      } catch (IllegalArgumentException e) {
        return DirectiveType.UNKNOWN;
      }
    }
  }

  private static void log(
      final Level level,
      final String message,
      final String robotsTxtBody,
      final int lineBegin,
      final int lineEnd,
      final int lineNumber) {
    logger.at(level).log(
        "%s%nAt line %d:%n%s\t", message, lineNumber, robotsTxtBody.substring(lineBegin, lineEnd));
  }

  private void parseLine(
      final String robotsTxtBody, final int lineBegin, final int lineEnd, final int lineNumber) {
    int limit = lineEnd;
    int separator = lineEnd;
    int whitespaceSeparator = lineEnd;
    boolean hasContents = false;

    for (int i = lineBegin; i < lineEnd; i++) {
      final char ch = robotsTxtBody.charAt(i);
      if (ch == '#') {
        limit = i;
        break;
      }
      if (!isWhitespace(ch)) {
        hasContents = true;
      }
      if (isWhitespace(ch) && hasContents) {
        whitespaceSeparator = i;
      }
      if (separator == lineEnd && ch == ':') {
        separator = i;
      }
    }

    if (separator == lineEnd) {
      // Google-specific optimization: some people forget the colon, so we need to
      // accept whitespace in its stead.
      if (whitespaceSeparator != lineEnd) {
        log(
            Level.INFO,
            "Assuming whitespace as a separator.",
            robotsTxtBody,
            lineBegin,
            lineEnd,
            lineNumber);
        separator = whitespaceSeparator;
      } else {
        if (hasContents) {
          log(Level.WARNING, "No separator found.", robotsTxtBody, lineBegin, lineEnd, lineNumber);
        }
        return;
      }
    }

    final String key;
{'fraction_non_alphanumeric': 0.0953160412994208, 'fraction_numerical': 0.002266431629312516, 'mean_word_length': 2.6385707741639943, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '372055', 'n_tokens_mistral': 2242, 'n_tokens_neox': 2178, 'n_words': 730}