<commit_before># frozen_string_literal: true

module RuboCop
  module Cop
    module Lint
      # This cop checks for quotes and commas in %w, e.g. `%w('foo', "bar")`
      #
      # It is more likely that the additional characters are unintended (for
      # example, mistranslating an array of literals to percent string notation)
      # rather than meant to be part of the resulting strings.
      #
      # @example
      #
      #   # bad
      #
      #   %w('foo', "bar")
      #
      # @example
      #
      #   # good
      #
      #   %w(foo bar)
      class PercentStringArray < Cop
        include PercentLiteral

        QUOTES_AND_COMMAS = [/,$/, /^'.*'$/, /^".*"$/].freeze
        LEADING_QUOTE = /^['"]/
        TRAILING_QUOTE = /['"]?,?$/

        MSG = "Within `%w`/`%W`, quotes and ',' are unnecessary and may be " \
          'unwanted in the resulting strings.'.freeze

        def on_array(node)
          process(node, '%w', '%W')
        end

        def on_percent_literal(node)
          return unless contains_quotes_or_commas?(node)

          add_offense(node)
        end

        private

        def contains_quotes_or_commas?(node)
          node.values.any? do |value|
            literal = scrub_string(value.children.first.to_s)

            # To avoid likely false positives (e.g. a single ' or ")
            next if literal.gsub(/[^\p{Alnum}]/, '').empty?

            QUOTES_AND_COMMAS.any? { |pat| literal =~ pat }
          end
        end

        # rubocop:disable Performance/HashEachMethods
        def autocorrect(node)
          lambda do |corrector|
            node.values.each do |value|
              range = value.loc.expression

              match = range.source.match(TRAILING_QUOTE)
              corrector.remove_trailing(range, match[0].length) if match

              if range.source =~ LEADING_QUOTE
                corrector.remove_leading(range, 1)
              end
            end
          end
        end
        # rubocop:enable Performance/HashEachMethod

        def scrub_string(string)
          if string.respond_to?(:scrub)
            string.scrub
          else
            string
              .encode('UTF-16BE', 'UTF-8',
                      invalid: :replace, undef: :replace, replace: '?')
              .encode('UTF-8')
          end
        end
      end
    end
  end
end<commit_msg>Remove duplicated scrub_string definition<commit_after># frozen_string_literal: true

module RuboCop
  module Cop
    module Lint
      # This cop checks for quotes and commas in %w, e.g. `%w('foo', "bar")`
      #
      # It is more likely that the additional characters are unintended (for
      # example, mistranslating an array of literals to percent string notation)
      # rather than meant to be part of the resulting strings.
      #
      # @example
      #
      #   # bad
      #
      #   %w('foo', "bar")
      #
      # @example
      #
      #   # good
      #
      #   %w(foo bar)
      class PercentStringArray < Cop
        include PercentLiteral

        QUOTES_AND_COMMAS = [/,$/, /^'.*'$/, /^".*"$/].freeze
        LEADING_QUOTE = /^['"]/
        TRAILING_QUOTE = /['"]?,?$/

        MSG = "Within `%w`/`%W`, quotes and ',' are unnecessary and may be " \
          'unwanted in the resulting strings.'.freeze

        def on_array(node)
          process(node, '%w', '%W')
        end

        def on_percent_literal(node)
          return unless contains_quotes_or_commas?(node)

          add_offense(node)
        end

        private

        def contains_quotes_or_commas?(node)
          node.values.any? do |value|
            literal = scrub_string(value.children.first.to_s)

            # To avoid likely false positives (e.g. a single ' or ")
            next if literal.gsub(/[^\p{Alnum}]/, '').empty?

            QUOTES_AND_COMMAS.any? { |pat| literal =~ pat }
          end
        end

        # rubocop:disable Performance/HashEachMethods
        def autocorrect(node)
          lambda do |corrector|
            node.values.each do |value|
              range = value.loc.expression

              match = range.source.match(TRAILING_QUOTE)
              corrector.remove_trailing(range, match[0].length) if match

              if range.source =~ LEADING_QUOTE
                corrector.remove_leading(range, 1)
              end
            end
          end
        end
        # rubocop:enable Performance/HashEachMethod
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.10638297872340426, 'fraction_numerical': 0.0017917133258678612, 'mean_word_length': 1.6838942307692308, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5385883', 'n_tokens_mistral': 1386, 'n_tokens_neox': 1318, 'n_words': 333}