<commit_before>//
//  Created by <NAME> on 3/22/16.
//  Copyright © 2016 Square, Inc. All rights reserved.
//  Released under the Apache v2 License.
//
//  Adapted from https://gist.github.com/blakemerryman/76312e1cbf8aec248167
//  Adapted from https://gist.github.com/efirestone/ce01ae109e08772647eb061b3bb387c3


import Foundation


public let GlobBehaviorBashV3 = Glob.Behavior(
  supportsGlobstar: false,
  includesFilesFromRootOfGlobstar: false,
  includesDirectoriesInResults: true,
  includesFilesInResultsIfTrailingSlash: false
)
public let GlobBehaviorBashV4 = Glob.Behavior(
  supportsGlobstar: true, // Matches Bash v4 with "shopt -s globstar" option
  includesFilesFromRootOfGlobstar: true,
  includesDirectoriesInResults: true,
  includesFilesInResultsIfTrailingSlash: false
)
public let GlobBehaviorGradle = Glob.Behavior(
  supportsGlobstar: true,
  includesFilesFromRootOfGlobstar: true,
  includesDirectoriesInResults: false,
  includesFilesInResultsIfTrailingSlash: true
)


/**
 Finds files on the file system using pattern matching.
 */
public class Glob: Collection {

  /**
   * Different glob implementations have different behaviors, so the behavior of this
   * implementation is customizable.
   */
  public struct Behavior {
    // If true then a globstar ("**") causes matching to be done recursively in subdirectories.
    // If false then "**" is treated the same as "*"
    let supportsGlobstar: Bool

    // If true the results from the directory where the globstar is declared will be included as well.
    // For example, with the pattern "dir/**/*.ext" the fie "dir/file.ext" would be included if this
    // property is true, and would be omitted if it's false.
    let includesFilesFromRootOfGlobstar: Bool

    // If false then the results will not include directory entries. This does not affect recursion depth.
    let includesDirectoriesInResults: Bool

    // If false and the last characters of the pattern are "**/" then only directories are returned in the results.
    let includesFilesInResultsIfTrailingSlash: Bool
  }

  static var defaultBehavior = GlobBehaviorBashV4

  private var isDirectoryCache = [String: Bool]()

  public let behavior: Behavior
  var paths = [String]()
  public var startIndex: Int { return paths.startIndex }
  public var endIndex: Int   { return paths.endIndex   }

  public init(pattern: String, behavior: Behavior = Glob.defaultBehavior) {

    self.behavior = behavior

    var adjustedPattern = pattern
    let hasTrailingGlobstarSlash = pattern.hasSuffix("**/")
    var includeFiles = !hasTrailingGlobstarSlash

    if behavior.includesFilesInResultsIfTrailingSlash {<commit_msg>Make property public for Swift 4<commit_after>//
//  Created by <NAME> on 3/22/16.
//  Copyright © 2016 Square, Inc. All rights reserved.
//  Released under the Apache v2 License.
//
//  Adapted from https://gist.github.com/blakemerryman/76312e1cbf8aec248167
//  Adapted from https://gist.github.com/efirestone/ce01ae109e08772647eb061b3bb387c3


import Foundation


public let GlobBehaviorBashV3 = Glob.Behavior(
  supportsGlobstar: false,
  includesFilesFromRootOfGlobstar: false,
  includesDirectoriesInResults: true,
  includesFilesInResultsIfTrailingSlash: false
)
public let GlobBehaviorBashV4 = Glob.Behavior(
  supportsGlobstar: true, // Matches Bash v4 with "shopt -s globstar" option
  includesFilesFromRootOfGlobstar: true,
  includesDirectoriesInResults: true,
  includesFilesInResultsIfTrailingSlash: false
)
public let GlobBehaviorGradle = Glob.Behavior(
  supportsGlobstar: true,
  includesFilesFromRootOfGlobstar: true,
  includesDirectoriesInResults: false,
  includesFilesInResultsIfTrailingSlash: true
)


/**
 Finds files on the file system using pattern matching.
 */
public class Glob: Collection {

  /**
   * Different glob implementations have different behaviors, so the behavior of this
   * implementation is customizable.
   */
  public struct Behavior {
    // If true then a globstar ("**") causes matching to be done recursively in subdirectories.
    // If false then "**" is treated the same as "*"
    let supportsGlobstar: Bool

    // If true the results from the directory where the globstar is declared will be included as well.
    // For example, with the pattern "dir/**/*.ext" the fie "dir/file.ext" would be included if this
    // property is true, and would be omitted if it's false.
    let includesFilesFromRootOfGlobstar: Bool

    // If false then the results will not include directory entries. This does not affect recursion depth.
    let includesDirectoriesInResults: Bool

    // If false and the last characters of the pattern are "**/" then only directories are returned in the results.
    let includesFilesInResultsIfTrailingSlash: Bool
  }

  public static var defaultBehavior = GlobBehaviorBashV4

  private var isDirectoryCache = [String: Bool]()

  public let behavior: Behavior
  var paths = [String]()
  public var startIndex: Int { return paths.startIndex }
  public var endIndex: Int   { return paths.endIndex   }

  public init(pattern: String, behavior: Behavior = Glob.defaultBehavior) {

    self.behavior = behavior

    var adjustedPattern = pattern
    let hasTrailingGlobstarSlash = pattern.hasSuffix("**/")
    var includeFiles = !hasTrailingGlobstarSlash

    if behavior.includesFilesInResultsIfTrailingSlash {
{'fraction_non_alphanumeric': 0.07479431563201197, 'fraction_numerical': 0.018137621540762903, 'mean_word_length': 4.0605487228003785, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4639374', 'n_tokens_mistral': 1625, 'n_tokens_neox': 1522, 'n_words': 520}