<commit_before>// Copyright 2010, <NAME>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.prebake.core;

import org.prebake.js.JsonSerializable;
import org.prebake.js.JsonSink;
import org.prebake.js.YSONConverter;

import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;

import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.Nullable;
import javax.annotation.ParametersAreNonnullByDefault;

/**
 * A pattern that matches a group of files.
 *
 * <p>Parts of the below talk about <b>normalized paths</b>.  Normalized paths
 * use {@code /} to separate individual file and directory names instead of
 * using the system dependent separator.  E.g., on Windows, the normalized form
 * of {@code C:\foo} is {@code C:/foo}.
 *
 * <h2>Tree Roots</h2>
 * A glob may specify that a prefix of all paths matched by it is the root
 * of a directory tree, such as <ul>
 *   <li>Portion of a C header file include directory,</li>
 *   <li>A java package tree,</li>
 *   <li>The root of files in a JAR or ZIP archive,</li>
 *   <li>The directory to contain the {@code index.html} file in a tree of
 *       HTML reports,</li>
 *   <li>etc.</li>
 * </ul>
 * The tree root of a glob is the prefix of it that precedes 3 path separators.
 * E.g. the tree root of {@code src///org/prebake/**.java} is {@code src}, and
 * the rest of the prefix, {@code /org/prebake} are path elements under the tree
 * root. That glob matches paths like {@code src/org/prebake/Foo.java} ; the
 * paths it matches do not have to have tripled file separators.
 *
 * @see <a href="http://code.google.com/p/prebake/wiki/Glob">Wiki Docs</a>
 * @author <NAME> <<EMAIL>>
 */
@ParametersAreNonnullByDefault
public final class Glob implements Comparable<Glob>, JsonSerializable {
  private final String[] parts;
  private final int treeRootIndex;
  private transient Pattern regex;
  // TODO: add the notion of a tree root to globs.
  // For example, the tree root of a set of java files might be the tree root.
  // Possible syntax
  // src///**.java
  // lib///com/google/**.class

  private Glob(int treeRootIndex, String... parts) {
    this.parts = parts;
    this.treeRootIndex = treeRootIndex;
  }

  /** See the grammar at http://code.google.com/p/prebake/wiki/Glob. */
  public static Glob fromString(String s) {
    int n = s.length();
    int partCount = 0;
    int treeRootIndex = 0;
    boolean sawStar = false;
    for (int i = 0; i < n; ++i) {
      ++partCount;
      switch (s.charAt(i)) {
        case '*':  // * or **
          if (i + 1 < n && s.charAt(i + 1) == '*') {
            ++i;
            // Three adjacent **'s not allowed.
            if (i + 1 < n && s.charAt(i + 1) == '*') { badGlob(s); }
          }
          sawStar = true;
          break;
        case '/': {
          // Two adjacent path separators not allowed unless as part of a tree
          // root index.
          int start = i;
          while (i + 1 < n && s.charAt(i + 1) == '/') { ++i; }
          switch (i - start) {  // number of extra slashes
            case 0: break;
            case 2:
              if (treeRootIndex == 0 && !sawStar) {
                treeRootIndex = partCount - 1;
                break;
              }
              // $FALL-THROUGH$
            default: badGlob(s);
          }
          break;
        }
        default:
          while (i + 1 < n) {
            char next = s.charAt(i + 1);
            if (next == '/') { break; }
            // * must follow / or start the glob.
            if (next == '*') { badGlob(s); }
            ++i;
          }<commit_msg>tweaked comments<commit_after>// Copyright 2010, <NAME>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.prebake.core;

import org.prebake.js.JsonSerializable;
import org.prebake.js.JsonSink;
import org.prebake.js.YSONConverter;

import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;

import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.Nullable;
import javax.annotation.ParametersAreNonnullByDefault;

/**
 * A pattern that matches a group of files.
 *
 * <p>Parts of the below talk about <b>normalized paths</b>.  Normalized paths
 * use {@code /} to separate individual file and directory names instead of
 * using the system dependent separator.  E.g., on Windows, the normalized form
 * of {@code C:\foo} is {@code C:/foo}.
 *
 * <h2>Tree Roots</h2>
 * A glob may specify that a prefix of all paths matched by it is the root
 * of a directory tree, such as <ul>
 *   <li>Portion of a C header file include directory,</li>
 *   <li>A java package tree,</li>
 *   <li>The root of files in a JAR or ZIP archive,</li>
 *   <li>The directory to contain the {@code index.html} file in a tree of
 *       HTML reports,</li>
 *   <li>etc.</li>
 * </ul>
 * The tree root of a glob is the prefix of it that precedes 3 path separators.
 * E.g. the tree root of {@code src///org/prebake/**.java} is {@code src}, and
 * the rest of the prefix, {@code /org/prebake} are path elements under the tree
 * root. That glob matches paths like {@code src/org/prebake/Foo.java} ; the
 * paths it matches do not have to have tripled file separators.
 *
 * @see <a href="http://code.google.com/p/prebake/wiki/Glob">Wiki Docs</a>
 * @author <NAME> <mikesamuel<EMAIL>>
 */
@ParametersAreNonnullByDefault
public final class Glob implements Comparable<Glob>, JsonSerializable {
  private final String[] parts;
  private final int treeRootIndex;
  private transient Pattern regex;

  private Glob(int treeRootIndex, String... parts) {
    this.parts = parts;
    this.treeRootIndex = treeRootIndex;
  }

  /** See the grammar at http://code.google.com/p/prebake/wiki/Glob. */
  public static Glob fromString(String s) {
    int n = s.length();
    int partCount = 0;
    int treeRootIndex = 0;
    boolean sawStar = false;
    for (int i = 0; i < n; ++i) {
      ++partCount;
      switch (s.charAt(i)) {
        case '*':  // * or **
          if (i + 1 < n && s.charAt(i + 1) == '*') {
            ++i;
            // Three adjacent *'s not allowed.
            if (i + 1 < n && s.charAt(i + 1) == '*') { badGlob(s); }
          }
          sawStar = true;
          break;
        case '/': {
          // Two adjacent path separators not allowed unless as part of a tree
          // root index.
          int start = i;
          while (i + 1 < n && s.charAt(i + 1) == '/') { ++i; }
          switch (i - start) {  // number of extra slashes
            case 0: break;
            case 2:
              if (treeRootIndex == 0 && !sawStar) {
                treeRootIndex = partCount - 1;
                break;
              }
              // $FALL-THROUGH$
            default: badGlob(s);
          }
          break;
        }
        default:
          while (i + 1 < n) {
            char next = s.charAt(i + 1);
            if (next == '/') { break; }
            // * must follow / or start the glob.
            if (next == '*') { badGlob(s); }
            ++i;
          }
{'fraction_non_alphanumeric': 0.12550051481523852, 'fraction_numerical': 0.005948976089692255, 'mean_word_length': 2.421526418786693, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '100780', 'n_tokens_mistral': 2727, 'n_tokens_neox': 2654, 'n_words': 939}