<commit_before>#
# Copyright 2014 Chef Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

require "fileutils" unless defined?(FileUtils)

module Omnibus
  module FileSyncer
    extend self

    # Files to be ignored during a directory globbing
    IGNORED_FILES = %w{. ..}.freeze

    #
    # Glob across the given pattern, accounting for dotfiles, removing Ruby's
    # dumb idea to include +'.'+ and +'..'+ as entries.
    #
    # @param [String] pattern
    #   the path or glob pattern to get all files from
    #
    # @return [Array<String>]
    #   the list of all files
    #
    def glob(pattern)
      pattern = Pathname.new(pattern).cleanpath.to_s
      Dir.glob(pattern, File::FNM_DOTMATCH).sort.reject do |file|
        basename = File.basename(file)
        IGNORED_FILES.include?(basename)
      end
    end

    #
    # Glob for all files under a given path/pattern, removing Ruby's
    # dumb idea to include +'.'+ and +'..'+ as entries.
    #
    # @param [String] source
    #   the path or glob pattern to get all files from
    #
    # @option options [String, Array<String>] :exclude
    #   a file, folder, or globbing pattern of files to ignore when syncing
    # @option options [String, Array<String>] :include
    #   a file, folder, or globbing pattern of files that has to be matched
    #   when syncing
    #
    # @return [Array<String>]
    #   the list of all files
    #
    def all_files_under(source, options = {})
      excludes = Array(options[:exclude]).map do |exclude|
        [exclude, "#{exclude}/**"]
      end.flatten

      includes = Array(options[:include]).map do |include|
        [include, "#{include}/**"]
      end.flatten

      source_files = glob(File.join(source, "**/*"))
      source_files = source_files.reject do |source_file|
        basename = relative_path_for(source_file, source)
        excludes.any? { |exclude| File.fnmatch?(exclude, basename, File::FNM_DOTMATCH | File::FNM_PATHNAME) }
      end

      if not includes.empty?
        source_files = source_files.reject do |source_file|
          basename = relative_path_for(source_file, source)
          includes.none? { |include| File.fnmatch?(include, basename, File::FNM_DOTMATCH | File::FNM_PATHNAME) }
        end
      end

      source_files
    end

    #
    # Copy the files from +source+ to +destination+, while removing any files
    # in +destination+ that are not present in +source+.
    #
    # The method accepts an optional +:exclude+ parameter to ignore files and
    # folders that match the given pattern(s). Note the exclude pattern behaves
    # on paths relative to the given source. If you want to exclude a nested
    # directory, you will need to use something like +**/directory+.
    #
    # @raise ArgumentError
    #   if the +source+ parameter is not a directory
    #
    # @param [String] source<commit_msg>Fix debug packages (#130)<commit_after>#
# Copyright 2014 Chef Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

require "fileutils" unless defined?(FileUtils)

module Omnibus
  module FileSyncer
    extend self

    # Files to be ignored during a directory globbing
    IGNORED_FILES = %w{. ..}.freeze

    #
    # Glob across the given pattern, accounting for dotfiles, removing Ruby's
    # dumb idea to include +'.'+ and +'..'+ as entries.
    #
    # @param [String] pattern
    #   the path or glob pattern to get all files from
    #
    # @return [Array<String>]
    #   the list of all files
    #
    def glob(pattern)
      pattern = Pathname.new(pattern).cleanpath.to_s
      Dir.glob(pattern, File::FNM_DOTMATCH).sort.reject do |file|
        basename = File.basename(file)
        IGNORED_FILES.include?(basename)
      end
    end

    #
    # Glob for all files under a given path/pattern, removing Ruby's
    # dumb idea to include +'.'+ and +'..'+ as entries.
    #
    # @param [String] source
    #   the path or glob pattern to get all files from
    #
    # @option options [String, Array<String>] :exclude
    #   a file, folder, or globbing pattern of files to ignore when syncing
    # @option options [String, Array<String>] :include
    #   a file, folder, or globbing pattern of files that has to be matched
    #   when syncing
    #
    # @return [Array<String>]
    #   the list of all files
    #
    def all_files_under(source, options = {})
      excludes = Array(options[:exclude]).map do |exclude|
        [exclude, "#{exclude}/**"]
      end.flatten

      includes = Array(options[:include]).map do |include|
        [include, "#{include}/**"]
      end.flatten

      source_files = glob(File.join(source, "**/*"))
      source_files = source_files.reject do |source_file|
        basename = relative_path_for(source_file, source)
        excludes.any? { |exclude| File.fnmatch?(exclude, basename, File::FNM_DOTMATCH | File::FNM_PATHNAME) }
      end

      if not includes.empty?
        source_files = source_files.reject do |source_file|
          basename = relative_path_for(source_file, source)
          # File::FNM_PATHNAME Prohibit wildcards from matching a slash ('/')
          # which means includes.none? will return true when the path includes a '/':
          #
          #   File.fnmatch?(".debug"), .debug/opt, File::FNM_DOTMATCH | File::FNM_PATHNAME) = false
          #   File.fnmatch?(".debug/**"), .debug/opt, File::FNM_DOTMATCH | File::FNM_PATHNAME) = true
          #   includes.none? { |include| File.fnmatch?(include), .debug/opt, File::FNM_DOTMATCH | File::FNM_PATHNAME) } = false
          #
          #   File.fnmatch?(".debug"), .debug/opt/datadog-agent, File::FNM_DOTMATCH | File::FNM_PATHNAME) = false
          #   File.fnmatch?(".debug/**"), .debug/opt/datadog-agent, File::FNM_DOTMATCH | File::FNM_PATHNAME) = false
          #   includes.none? { |include| File.fnmatch?(include), .debug/opt/datadog-agent, File::FNM_DOTMATCH | File::FNM_PATHNAME) } = true
          #
          # As per above, we can see that it would not include any subfolder beneath the fist level of nesting.
          includes.none? { |include| File.fnmatch?(include, basename, File::FNM_DOTMATCH) }
        end
      end

      source_files
    end

    #
    # Copy the files from +source+ to +destination+, while removing any files
    # in +destination+ that are not present in +source+.
    #
    # The method accepts an optional +:exclude+ parameter to ignore files and
    # folders that match the given pattern(s). Note the exclude pattern behaves
    # on paths relative to the given source. If you want to exclude a nested
    # directory, you will need to use something like +**/directory+.
    #
    # @raise ArgumentError
    #   if the +source+ parameter is not a directory
    #
    # @param [String] source
{'fraction_non_alphanumeric': 0.10478154401340378, 'fraction_numerical': 0.0024487691712849593, 'mean_word_length': 2.8703241895261846, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2820539', 'n_tokens_mistral': 2324, 'n_tokens_neox': 2252, 'n_words': 790}