<commit_before>require 'bundler/setup'
require 'nokogiri'

module ObjectiveCi
    class CiTasks

    attr_accessor :exclusions

    LINT_DESTINATION = "lint.xml"
    DUPLICATION_DESTINATION = "duplication.xml"
    LINE_COUNT_DESTINATION = "line-count.sc"    

    def initialize
      @exclusions = ["vendor"]
      if using_pods?
        @exclusions << "Pods"
        `bundle exec pod install`
      end
    end

    def build(opts={})
      lint(opts)
      lines_of_code(opts)
      test_suite(opts)
      duplicate_code_detection(opts)
    end

    def lint(opts={})
      requires_at_least_one_option(opts, :workspace, :project)
      requires_options(opts, :scheme)
      opts[:configuration] ||= "Release"

      sliced_opts = opts.select { |k, v| [:scheme, :workspace, :project, :configuration].include?(k) }
      xcodebuild_opts_string = sliced_opts.reduce("") { |str, (k, v)| str += " -#{k} #{v}" }
      xcodebuild_opts_string += " ONLY_ACTIVE_ARCH=NO clean build"

      call_binary("xcodebuild", xcodebuild_opts_string, "| tee xcodebuild.log", opts)
      # oclint-xcodebuild will fail if we don't exclude Pods (absolute path)
      pods_dir = "#{Dir.pwd}/Pods"
      call_binary("oclint-xcodebuild", "-e \"#{pods_dir}\"", "", opts)
      ocjcd_opts_string = "#{exclusion_options_list("-e")} -- -report-type=pmd -o=#{LINT_DESTINATION}"
      call_binary("oclint-json-compilation-database", ocjcd_opts_string, "", opts)
    end

    def test_suite(opts={})
      requires_at_least_one_option(opts, :workspace, :project)
      requires_options(opts ,:scheme)
      if !opts[:xcodebuild_override] && xcode_version < 5.0
        puts_red "WARNING: Xcode version #{xcode_version} is less than 5.0, and tests will likely not run"
      end
      
      sliced_opts = opts.select { |k, v| [:scheme, :workspace, :project].include?(k) }
      xcodebuild_opts_string = sliced_opts.reduce("") { |str, (k, v)| str += " -#{k} #{v}" }

      xcodebuild_opts_string += " -destination name=iPad -destination-timeout=10 ONLY_ACTIVE_ARCH=NO test"
      call_binary("xcodebuild", xcodebuild_opts_string, ">&1 | bundle exec ocunit2junit", opts)
    end

    def lines_of_code(opts={})
      call_binary("sloccount",
                  "--duplicates --wide --details .",
                  "| grep -v #{exclusion_options_list("-e")} > #{LINE_COUNT_DESTINATION}", 
                  opts)
    end

    def duplicate_code_detection(opts={})
      opts[:minimum_tokens] ||= 100
      # Use `sed` to change paths like `/some/code/./path.m` to `/some/code/path.m`, or else the Violations plugin in Jenkins
      # doesn't work correctly.
      call_binary("pmd-cpd-objc",
                  "--minimum-tokens #{opts[:minimum_tokens]}",
                  "| LC_CTYPE=C LANG=C sed 's/\\/\\.\\//\\//' > #{DUPLICATION_DESTINATION}", 
                  opts)
      pmd_exclude
    end

    private
    def exclusion_options_list(option_flag)
      if exclusions.empty?
        ''
      else
        wrapped_exclusions = exclusions.map { |e| "\"#{e}\"" }
        "#{option_flag} #{wrapped_exclusions.join(" #{option_flag} ")}"
      end
    end

    private
    def using_pods?
      File.exists?("Podfile")
    end

    private
    def call_binary(binary, cl_options, tail, opts={})
      extra_options = opts["#{binary}_options".to_sym]
      override_options = opts["#{binary}_override".to_sym]
      cl_options = override_options ? extra_options : "#{cl_options} #{extra_options}"
      command = "#{binary} #{cl_options} #{tail}"
      command.prepend("bundle exec ") unless binary == "xcodebuild"
      puts command
      `#{command}`
    end

    private 
    def requires_options(opts, *keys)
      keys.each do |k|
        raise "option #{k} is required." unless opts.has_key?(k)
      end
    end

    private
    def requires_at_least_one_option(opts, *keys)
      if (opts.keys && keys).empty?
        raise "at least one of the options #{keys.join(", ")} is required"
      end
    end

    private
    def pmd_exclude
      # Unfortunately, pmd doesn't seem to provide any nice out-of-the-box way for excluding files from the results.
      absolute_exclusions = exclusions.map { |e| "#{Dir.pwd}/./#{e}/" }
      regex_exclusion = Regexp.new("(#{absolute_exclusions.join("|")})")
      output = Nokogiri::XML(File.open(DUPLICATION_DESTINATION))
      output.xpath("//duplication").each do |duplication_node|
        if duplication_node.xpath("file").all? { |n| n["path"] =~ regex_exclusion }
          duplication_node.remove
        end
      end
      File.open(DUPLICATION_DESTINATION, 'w') { |file| file.write(output.to_s) }
    end

    private
    def xcode_version
      matches = `xcodebuild -version`.match(/^Xcode ([0-9]+\.[0-9]+)/)
      matches ? matches[1].to_f : 0.0
    end

    private 
    def puts_red(str)
      puts "\e[31m#{str}\e[0m"
    end
  end
end<commit_msg>Removed these with sed<commit_after>require 'bundler/setup'
require 'nokogiri'

module ObjectiveCi
    class CiTasks

    attr_accessor :exclusions

    LINT_DESTINATION = "lint.xml"
    DUPLICATION_DESTINATION = "duplication.xml"
    LINE_COUNT_DESTINATION = "line-count.sc"    

    def initialize
      @exclusions = ["vendor"]
      if using_pods?
        @exclusions << "Pods"
        `bundle exec pod install`
      end
    end

    def build(opts={})
      lint(opts)
      lines_of_code(opts)
      test_suite(opts)
      duplicate_code_detection(opts)
    end

    def lint(opts={})
      requires_at_least_one_option(opts, :workspace, :project)
      requires_options(opts, :scheme)
      opts[:configuration] ||= "Release"

      sliced_opts = opts.select { |k, v| [:scheme, :workspace, :project, :configuration].include?(k) }
      xcodebuild_opts_string = sliced_opts.reduce("") { |str, (k, v)| str += " -#{k} #{v}" }
      xcodebuild_opts_string += " ONLY_ACTIVE_ARCH=NO clean build"

      call_binary("xcodebuild", xcodebuild_opts_string, "| tee xcodebuild.log", opts)
      # oclint-xcodebuild will fail if we don't exclude Pods (absolute path)
      pods_dir = "#{Dir.pwd}/Pods"
      call_binary("oclint-xcodebuild", "-e \"#{pods_dir}\"", "", opts)
      ocjcd_opts_string = "#{exclusion_options_list("-e")} -- -report-type=pmd -o=#{LINT_DESTINATION}"
      call_binary("oclint-json-compilation-database", ocjcd_opts_string, "", opts)
    end

    def test_suite(opts={})
      requires_at_least_one_option(opts, :workspace, :project)
      requires_options(opts ,:scheme)
      if !opts[:xcodebuild_override] && xcode_version < 5.0
        puts_red "WARNING: Xcode version #{xcode_version} is less than 5.0, and tests will likely not run"
      end
      
      sliced_opts = opts.select { |k, v| [:scheme, :workspace, :project].include?(k) }
      xcodebuild_opts_string = sliced_opts.reduce("") { |str, (k, v)| str += " -#{k} #{v}" }

      xcodebuild_opts_string += " -destination name=iPad -destination-timeout=10 ONLY_ACTIVE_ARCH=NO test"
      call_binary("xcodebuild", xcodebuild_opts_string, ">&1 | bundle exec ocunit2junit", opts)
    end

    def lines_of_code(opts={})
      call_binary("sloccount",
                  "--duplicates --wide --details .",
                  "| grep -v #{exclusion_options_list("-e")} > #{LINE_COUNT_DESTINATION}", 
                  opts)
    end

    def duplicate_code_detection(opts={})
      opts[:minimum_tokens] ||= 100
      # Use `sed` to change paths like `/some/code/./path.m` to `/some/code/path.m`, or else the Violations plugin in Jenkins
      # doesn't work correctly.
      call_binary("pmd-cpd-objc",
                  "--minimum-tokens #{opts[:minimum_tokens]}",
                  "| LC_CTYPE=C LANG=C sed 's/\\/\\.\\//\\//' > #{DUPLICATION_DESTINATION}", 
                  opts)
      pmd_exclude
    end

    private
    def exclusion_options_list(option_flag)
      if exclusions.empty?
        ''
      else
        wrapped_exclusions = exclusions.map { |e| "\"#{e}\"" }
        "#{option_flag} #{wrapped_exclusions.join(" #{option_flag} ")}"
      end
    end

    private
    def using_pods?
      File.exists?("Podfile")
    end

    private
    def call_binary(binary, cl_options, tail, opts={})
      extra_options = opts["#{binary}_options".to_sym]
      override_options = opts["#{binary}_override".to_sym]
      cl_options = override_options ? extra_options : "#{cl_options} #{extra_options}"
      command = "#{binary} #{cl_options} #{tail}"
      command.prepend("bundle exec ") unless binary == "xcodebuild"
      puts command
      `#{command}`
    end

    private 
    def requires_options(opts, *keys)
      keys.each do |k|
        raise "option #{k} is required." unless opts.has_key?(k)
      end
    end

    private
    def requires_at_least_one_option(opts, *keys)
      if (opts.keys && keys).empty?
        raise "at least one of the options #{keys.join(", ")} is required"
      end
    end

    private
    def pmd_exclude
      # Unfortunately, pmd doesn't seem to provide any nice out-of-the-box way for excluding files from the results.
      absolute_exclusions = exclusions.map { |e| "#{Dir.pwd}/#{e}/" }
      regex_exclusion = Regexp.new("(#{absolute_exclusions.join("|")})")
      output = Nokogiri::XML(File.open(DUPLICATION_DESTINATION))
      output.xpath("//duplication").each do |duplication_node|
        if duplication_node.xpath("file").all? { |n| n["path"] =~ regex_exclusion }
          duplication_node.remove
        end
      end
      File.open(DUPLICATION_DESTINATION, 'w') { |file| file.write(output.to_s) }
    end

    private
    def xcode_version
      matches = `xcodebuild -version`.match(/^Xcode ([0-9]+\.[0-9]+)/)
      matches ? matches[1].to_f : 0.0
    end

    private 
    def puts_red(str)
      puts "\e[31m#{str}\e[0m"
    end
  end
end
{'fraction_non_alphanumeric': 0.14073542968349892, 'fraction_numerical': 0.004301956365871146, 'mean_word_length': 2.7933177933177933, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4795717', 'n_tokens_mistral': 3366, 'n_tokens_neox': 3247, 'n_words': 686}