<commit_before>#!/usr/bin/env ruby
$LOAD_PATH << File.join(File.expand_path(File.dirname(File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__)), 'lib')

require 'fileutils'
require 'open-uri'
require 'thread'
require 'trollop'
require 'yaml'
require 'tmpdir'

USAGE = <<END
Usage : scald.rb [options] job <job args>

  If job ends in ".scala" or ".java" and the file exists, then link
  it against JARFILE (default: versioned scalding-core jar) and run
  it (default: on HOST).  Otherwise, it is assumed to be a full
  classname to an item in the JARFILE, which is run.

END

##############################################################
# Default configuration:
  #Get the absolute path of the original (non-symlink) file.
CONFIG_DEFAULT = begin
  original_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
  repo_root = File.expand_path(File.dirname(original_file)+"/../")
  { "host" => "my.host.here", #where the job is rsynced to and run
    "repo_root" => repo_root, #full path to the repo you use, Twitter specific
    "cp" => ENV['CLASSPATH'] || "",
    "localmem" => "3g", #how much memory for java to use when running in local mode
    "namespaces" => { "abj" => "com.twitter.ads.batch.job", "s" => "com.twitter.scalding" },
    "hadoop_opts" => { "mapred.reduce.tasks" => 20, #be conservative by default
                       "mapred.min.split.size" => "2000000000" }, #2 billion bytes!!!
    "depends" => [ "org.apache.hadoop/hadoop-core/1.1.2",
                   "commons-codec/commons-codec/1.8",
                   "commons-configuration/commons-configuration/1.9",
                   "org.codehaus.jackson/jackson-asl/0.9.5",
                   "org.codehaus.jackson/jackson-mapper-asl/1.9.13",
                   "commons-lang/commons-lang/2.6",
                   "org.slf4j/slf4j-log4j12/1.6.6",
                   "log4j/log4j/1.2.15",
                   "commons-httpclient/commons-httpclient/3.1",
                   "commons-cli/commons-cli/1.2",
                   "commons-logging/commons-logging/1.1.1",
                   "org.apache.zookeeper/zookeeper/3.3.4" ],
    "default_mode" => "--hdfs"
  }
end
##############################################################

CONFIG_RC = begin
#put configuration in .scaldrc in HOME to override the defaults below:
    YAML.load_file(ENV['HOME'] + "/.scaldrc") || {} #seems that on ruby 1.9, this returns false on failure
  rescue
    {}
  end

CONFIG = CONFIG_DEFAULT.merge!(CONFIG_RC)

BUILDFILE = open(CONFIG["repo_root"] + "/project/Build.scala").read
VERSIONFILE = open(CONFIG["repo_root"] + "/version.sbt").read
SCALDING_VERSION=VERSIONFILE.match(/version.*:=\s*\"([^\"]+)\"/)[1]

#optionally set variables (not linux often doesn't have this set, and falls back to TMP. Set up a
#YAML file in .scaldrc with "tmpdir: my_tmp_directory_name" or export TMPDIR="/my/tmp" to set on
#linux
TMPDIR=CONFIG["tmpdir"] || ENV['TMPDIR'] || "/tmp"
TMPMAVENDIR = File.join(TMPDIR, "maven")
BUILDDIR=CONFIG["builddir"] || File.join(TMPDIR,"script-build")
LOCALMEM=CONFIG["localmem"] || "3g"
DEPENDENCIES=CONFIG["depends"] || []
RSYNC_STATFILE_PREFIX = TMPDIR + "/scald.touch."

#Recall that usage is of the form scald.rb [--jar jarfile] [--hdfs|--hdfs-local|--local|--print] [--print_cp] [--scalaversion version] job <job args>
#This parser holds the {job <job args>} part of the command.
OPTS_PARSER = Trollop::Parser.new do
  opt :clean, "Clean all rsync and maven state before running"
  opt :cp, "Scala classpath", :type => String
  opt :hdfs, "Run on HDFS"
  opt :hdfs_local, "Run in Hadoop local mode"
  opt :local, "Run in Cascading local mode (does not use Hadoop)"
  opt :print, "Print the command YOU SHOULD enter on the remote node. Useful for screen sessions"
  opt :scalaversion, "version of Scala for scalac (defaults to scalaVersion in project/Build.scala)", :type => String
  opt :print_cp, "Print the Scala classpath"
  opt :jar, "Specify the jar file", :type => String
  opt :host, "Specify the hadoop host where the job runs", :type => String
  opt :reducers, "Specify the number of reducers", :type => :int
  opt :avro, "Add scalding-avro to classpath"
  opt :commons, "Add scalding-commons to classpath"
  opt :jdbc, "Add scalding-jdbc to classpath"
  opt :json, "Add scalding-json to classpath"
  opt :parquet, "Add scalding-parquet to classpath"
  opt :repl, "Add scalding-repl to classpath"
  opt :tool, "The scalding main class, defaults to com.twitter.scalding.Tool", :type => String

  stop_on_unknown #Stop parsing for options parameters once we reach the job file.
end

#OPTS holds the option parameters that come before {job}, i.e., the
#[--jar jarfile] [--hdfs|--hdfs-local|--local|--print] part of the command.
OPTS =  Trollop::with_standard_exception_handling OPTS_PARSER do
  OPTS_PARSER.parse ARGV
end

#Make sure one of the execution modes is set.
unless [OPTS[:hdfs], OPTS[:hdfs_local], OPTS[:local], OPTS[:print]].any?
  #Modes in CONFIG file are in the form "--hdfs" or "--local", but the OPTS hash assumes
  #them to be in the form :hdfs or :local.
  #TODO: Add error checking on CONFIG["default_mode"]?
  mode = CONFIG["default_mode"] ? CONFIG["default_mode"].gsub("--", "").to_sym : :hdfs
  OPTS[mode] = true
end

def maven_filename(jar_filename)
  File.join(TMPMAVENDIR, jar_filename)
end

if OPTS[:clean]
  rsync_files = Dir.glob(RSYNC_STATFILE_PREFIX + "*")
  $stderr.puts("Cleaning rsync stat files: #{rsync_files.join(', ')}")
  rsync_files.each { |f| File.delete(f) }

  maven_files = Dir.glob(maven_filename("*"))
  $stderr.puts("Cleaning maven jars: #{maven_files.join(', ')}")
  maven_files.each { |f| File.delete(f) }
  Dir.rmdir(TMPMAVENDIR) if File.directory?(TMPMAVENDIR)
  #HACK -- exit immediately because other parts of this script assume more
  #arguments are passed in.
  exit(0)
end

if ARGV.size < 1 && OPTS[:repl].nil?
  $stderr.puts USAGE
  OPTS_PARSER::educate
  exit(0)
end

SCALA_VERSION= OPTS[:scalaversion] || BUILDFILE.match(/scalaVersion\s*:=\s*\"([^\"]+)\"/)[1]

SBT_HOME="#{ENV['HOME']}/.sbt"

SCALA_LIB_DIR="#{SBT_HOME}/boot/scala-#{SCALA_VERSION}/lib"

def scala_libs(version)
  if( version.start_with?("2.10") )
    ["scala-library", "scala-reflect", "scala-compiler"]
  else
    ["scala-library", "scala-compiler"]
  end
end

def find_dependencies(org, dep, version)
  res = %x[./sbt 'set libraryDependencies := Seq("#{org}" % "#{dep}" % "#{version}")' 'printDependencyClasspath'].split("\n")
  mapVer = {}
  res.map { |l|
    l,m,r = l.partition(" => ")
    if (m == " => ")
      removedSome = l.sub(/Some\(/, '').sub(/\)$/,'')
      mapVer[removedSome] = r
    else
      []
    end
  }

  mapVer
end

def find_dependency(org, reqDep, version)
  retDeps = find_dependencies(org, reqDep, version)
  dep = retDeps["#{org}:#{reqDep}:#{version}"]
  raise "Dependency #{org}:#{reqDep}:#{version} not found\n#{retDeps}" unless dep
  dep
end
<commit_msg>Handle 2.10.4 library resolution changes.<commit_after>#!/usr/bin/env ruby
$LOAD_PATH << File.join(File.expand_path(File.dirname(File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__)), 'lib')

require 'fileutils'
require 'open-uri'
require 'thread'
require 'trollop'
require 'yaml'
require 'tmpdir'

USAGE = <<END
Usage : scald.rb [options] job <job args>

  If job ends in ".scala" or ".java" and the file exists, then link
  it against JARFILE (default: versioned scalding-core jar) and run
  it (default: on HOST).  Otherwise, it is assumed to be a full
  classname to an item in the JARFILE, which is run.

END

##############################################################
# Default configuration:
  #Get the absolute path of the original (non-symlink) file.
CONFIG_DEFAULT = begin
  original_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
  repo_root = File.expand_path(File.dirname(original_file)+"/../")
  { "host" => "my.host.here", #where the job is rsynced to and run
    "repo_root" => repo_root, #full path to the repo you use, Twitter specific
    "cp" => ENV['CLASSPATH'] || "",
    "localmem" => "3g", #how much memory for java to use when running in local mode
    "namespaces" => { "abj" => "com.twitter.ads.batch.job", "s" => "com.twitter.scalding" },
    "hadoop_opts" => { "mapred.reduce.tasks" => 20, #be conservative by default
                       "mapred.min.split.size" => "2000000000" }, #2 billion bytes!!!
    "depends" => [ "org.apache.hadoop/hadoop-core/1.1.2",
                   "commons-codec/commons-codec/1.8",
                   "commons-configuration/commons-configuration/1.9",
                   "org.codehaus.jackson/jackson-asl/0.9.5",
                   "org.codehaus.jackson/jackson-mapper-asl/1.9.13",
                   "commons-lang/commons-lang/2.6",
                   "org.slf4j/slf4j-log4j12/1.6.6",
                   "log4j/log4j/1.2.15",
                   "commons-httpclient/commons-httpclient/3.1",
                   "commons-cli/commons-cli/1.2",
                   "commons-logging/commons-logging/1.1.1",
                   "org.apache.zookeeper/zookeeper/3.3.4" ],
    "default_mode" => "--hdfs"
  }
end
##############################################################

CONFIG_RC = begin
#put configuration in .scaldrc in HOME to override the defaults below:
    YAML.load_file(ENV['HOME'] + "/.scaldrc") || {} #seems that on ruby 1.9, this returns false on failure
  rescue
    {}
  end

CONFIG = CONFIG_DEFAULT.merge!(CONFIG_RC)

BUILDFILE = open(CONFIG["repo_root"] + "/project/Build.scala").read
VERSIONFILE = open(CONFIG["repo_root"] + "/version.sbt").read
SCALDING_VERSION=VERSIONFILE.match(/version.*:=\s*\"([^\"]+)\"/)[1]

#optionally set variables (not linux often doesn't have this set, and falls back to TMP. Set up a
#YAML file in .scaldrc with "tmpdir: my_tmp_directory_name" or export TMPDIR="/my/tmp" to set on
#linux
TMPDIR=CONFIG["tmpdir"] || ENV['TMPDIR'] || "/tmp"
TMPMAVENDIR = File.join(TMPDIR, "maven")
BUILDDIR=CONFIG["builddir"] || File.join(TMPDIR,"script-build")
LOCALMEM=CONFIG["localmem"] || "3g"
DEPENDENCIES=CONFIG["depends"] || []
RSYNC_STATFILE_PREFIX = TMPDIR + "/scald.touch."

#Recall that usage is of the form scald.rb [--jar jarfile] [--hdfs|--hdfs-local|--local|--print] [--print_cp] [--scalaversion version] job <job args>
#This parser holds the {job <job args>} part of the command.
OPTS_PARSER = Trollop::Parser.new do
  opt :clean, "Clean all rsync and maven state before running"
  opt :cp, "Scala classpath", :type => String
  opt :hdfs, "Run on HDFS"
  opt :hdfs_local, "Run in Hadoop local mode"
  opt :local, "Run in Cascading local mode (does not use Hadoop)"
  opt :print, "Print the command YOU SHOULD enter on the remote node. Useful for screen sessions"
  opt :scalaversion, "version of Scala for scalac (defaults to scalaVersion in project/Build.scala)", :type => String
  opt :print_cp, "Print the Scala classpath"
  opt :jar, "Specify the jar file", :type => String
  opt :host, "Specify the hadoop host where the job runs", :type => String
  opt :reducers, "Specify the number of reducers", :type => :int
  opt :avro, "Add scalding-avro to classpath"
  opt :commons, "Add scalding-commons to classpath"
  opt :jdbc, "Add scalding-jdbc to classpath"
  opt :json, "Add scalding-json to classpath"
  opt :parquet, "Add scalding-parquet to classpath"
  opt :repl, "Add scalding-repl to classpath"
  opt :tool, "The scalding main class, defaults to com.twitter.scalding.Tool", :type => String

  stop_on_unknown #Stop parsing for options parameters once we reach the job file.
end

#OPTS holds the option parameters that come before {job}, i.e., the
#[--jar jarfile] [--hdfs|--hdfs-local|--local|--print] part of the command.
OPTS =  Trollop::with_standard_exception_handling OPTS_PARSER do
  OPTS_PARSER.parse ARGV
end

#Make sure one of the execution modes is set.
unless [OPTS[:hdfs], OPTS[:hdfs_local], OPTS[:local], OPTS[:print]].any?
  #Modes in CONFIG file are in the form "--hdfs" or "--local", but the OPTS hash assumes
  #them to be in the form :hdfs or :local.
  #TODO: Add error checking on CONFIG["default_mode"]?
  mode = CONFIG["default_mode"] ? CONFIG["default_mode"].gsub("--", "").to_sym : :hdfs
  OPTS[mode] = true
end

def maven_filename(jar_filename)
  File.join(TMPMAVENDIR, jar_filename)
end

if OPTS[:clean]
  rsync_files = Dir.glob(RSYNC_STATFILE_PREFIX + "*")
  $stderr.puts("Cleaning rsync stat files: #{rsync_files.join(', ')}")
  rsync_files.each { |f| File.delete(f) }

  maven_files = Dir.glob(maven_filename("*"))
  $stderr.puts("Cleaning maven jars: #{maven_files.join(', ')}")
  maven_files.each { |f| File.delete(f) }
  Dir.rmdir(TMPMAVENDIR) if File.directory?(TMPMAVENDIR)
  #HACK -- exit immediately because other parts of this script assume more
  #arguments are passed in.
  exit(0)
end

if ARGV.size < 1 && OPTS[:repl].nil?
  $stderr.puts USAGE
  OPTS_PARSER::educate
  exit(0)
end

SCALA_VERSION= OPTS[:scalaversion] || BUILDFILE.match(/scalaVersion\s*:=\s*\"([^\"]+)\"/)[1]

SBT_HOME="#{ENV['HOME']}/.sbt"

SCALA_LIB_DIR="#{SBT_HOME}/boot/scala-#{SCALA_VERSION}/lib"

def scala_libs(version)
  if( version.start_with?("2.10") )
    ["scala-library", "scala-reflect", "scala-compiler"]
  else
    ["scala-library", "scala-compiler"]
  end
end

def find_dependencies(org, dep, version)
  res = %x[./sbt 'set libraryDependencies := Seq("#{org}" % "#{dep}" % "#{version}")' 'printDependencyClasspath'].split("\n")
  mapVer = {}
  res.map { |l|
    l,m,r = l.partition(" => ")
    if (m == " => ")
      removedSome = l.sub(/Some\(/, '').sub(/\)$/,'')
      removeExtraBraces = removedSome.sub(/ .*/, '') # In 2.10.4 for resolution for some reason there is a " ()" at the end
      mapVer[removeExtraBraces] = r
    else
      []
    end
  }

  mapVer
end

def find_dependency(org, reqDep, version)
  retDeps = find_dependencies(org, reqDep, version)
  dep = retDeps["#{org}:#{reqDep}:#{version}"]
  raise "Dependency #{org}:#{reqDep}:#{version} not found\n#{retDeps}" unless dep
  dep
end

{'fraction_non_alphanumeric': 0.17098112128146453, 'fraction_numerical': 0.009868421052631578, 'mean_word_length': 3.7584212317114667, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6110340', 'n_tokens_mistral': 4960, 'n_tokens_neox': 4814, 'n_words': 1289}