<commit_before>require 'java'
require 'tempfile'

# HBase connection
class HBase
  class << self
    # Resolve Hadoop and HBase dependency with Maven or hbase command (Experimental)
    # @param [String] dist Distribution version or path to pom.xml file
    # @param [true, false] verbose Verbose output
    # @return [Array<String>] Loaded JAR files
    def resolve_dependency! dist, verbose = false
      silencer = verbose ? '' : '> /dev/null'
      jars =
        if dist == :hbase
          # Check for hbase executable
          hbase = `which hbase`
          raise RuntimeError, "Cannot find executable `hbase`" if hbase.empty?
          `hbase classpath`.split(':')
        else
          # Check for Maven executable
          mvn = `which mvn`
          raise RuntimeError, "Cannot find executable `mvn`" if mvn.empty?

          path = [
            dist.to_s,
            File.expand_path("../pom/#{dist.downcase}.xml", __FILE__)
          ].select { |f| File.exists? f }.first

          # TODO: remote fetch from github head
          raise ArgumentError, "Invalid distribution: #{dist}" unless path

          # Download dependent JAR files and build classpath string
          tf = Tempfile.new 'hbase-jruby-classpath'
          tf.close(false) # Unlink later
          system "mvn org.apache.maven.plugins:maven-dependency-plugin:2.5.1:resolve org.apache.maven.plugins:maven-dependency-plugin:2.5.1:build-classpath -Dsilent=true -Dmdep.outputFile=#{tf.path} -f #{path} #{silencer}"
          File.read(tf.path).split(':')
        end

      # Load jars
      jars.select { |jar| File.exists?(jar) && File.extname(jar) == '.jar' }.select do |jar|
        require jar
      end
    ensure
      tf.unlink if tf
    end
  end
end<commit_msg>download pom.xml from github head<commit_after>require 'java'
require 'open-uri'
require 'tempfile'

# HBase connection
class HBase
  class << self
    # Resolve Hadoop and HBase dependency with Maven or hbase command (Experimental)
    # @param [String] dist Distribution version or path to pom.xml file
    # @param [true, false] verbose Verbose output
    # @return [Array<String>] Loaded JAR files
    def resolve_dependency! dist, verbose = false
      silencer = verbose ? '' : '> /dev/null'
      tfs = []
      jars =
        if dist == :hbase
          # Check for hbase executable
          hbase = `which hbase`
          raise RuntimeError, "Cannot find executable `hbase`" if hbase.empty?
          `hbase classpath`.split(':')
        else
          # Check for Maven executable
          mvn = `which mvn`
          raise RuntimeError, "Cannot find executable `mvn`" if mvn.empty?

          distname = dist.downcase.sub(/\.xml$/, '')
          path = [
            File.expand_path("../pom/#{distname}.xml", __FILE__),
            dist.to_s,
          ].select { |f| File.exists? f }.first

          # Try github head
          unless path
            begin
              xml = open("https://raw.github.com/junegunn/hbase-jruby/master/lib/hbase-jruby/pom/#{distname}.xml").read
              tfs << tf = Tempfile.new("#{distname}.xml")
              tf.close(false)
              path = tf.path
              File.open(path, 'w') do |f|
                f << xml
              end
            rescue OpenURI::HTTPError => e
              # No such distribution anywhere
            end
          end

          raise ArgumentError, "Invalid distribution: #{dist}" unless path

          # Download dependent JAR files and build classpath string
          tfs << tf = Tempfile.new('hbase-jruby-classpath')
          tf.close(false)
          system "mvn org.apache.maven.plugins:maven-dependency-plugin:2.5.1:resolve org.apache.maven.plugins:maven-dependency-plugin:2.5.1:build-classpath -Dsilent=true -Dmdep.outputFile=#{tf.path} -f #{path} #{silencer}"

          raise RuntimeError.new("Error occurred. Set verbose parameter to see the log.") unless $?.exitstatus == 0

          File.read(tf.path).split(':')
        end

      # Load jars
      jars.select { |jar| File.exists?(jar) && File.extname(jar) == '.jar' }.select do |jar|
        require jar
      end
    ensure
      tfs.each { |tf| tf.unlink rescue nil }
    end
  end
end
{'fraction_non_alphanumeric': 0.11612443600094989, 'fraction_numerical': 0.0030871526953217764, 'mean_word_length': 2.157421289355322, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3490837', 'n_tokens_mistral': 1297, 'n_tokens_neox': 1237, 'n_words': 357}