<commit_before>require 'albino/process'
##
# Wrapper for the Pygments command line tool, pygmentize.
#
# Pygments: http://pygments.org/
#
# Assumes pygmentize is in the path.  If not, set its location
# with Albino.bin = '/path/to/pygmentize'
#
# Use like so:
#
#   @syntaxer = Albino.new('/some/file.rb', :ruby)
#   puts @syntaxer.colorize
#
# This'll print out an HTMLized, Ruby-highlighted version
# of '/some/file.rb'.
#
# To use another formatter, pass it as the third argument:
#
#   @syntaxer = Albino.new('/some/file.rb', :ruby, :bbcode)
#   puts @syntaxer.colorize
#
# You can also use the #colorize class method:
#
#   puts Albino.colorize('/some/file.rb', :ruby)
#
# Another also: you get a #to_s, for somewhat nicer use in Rails views.
#
#   ... helper file ...
#   def highlight(text)
#     Albino.new(text, :ruby)
#   end
#
#   ... view file ...
#   <%= highlight text %>
#
# The default lexer is 'text'.  You need to specify a lexer yourself;
# because we are using STDIN there is no auto-detect.
#
# To see all lexers and formatters available, run `pygmentize -L`.
#
# <NAME> // <EMAIL>
#         GitHub // http://github.com
#
class Albino
  class ShellArgumentError < ArgumentError; end

  VERSION = '1.2.2'

  class << self
    attr_accessor :bin, :default_encoding, :timeout_threshold
  end
  self.default_encoding = 'utf8'
  self.bin = 'pygmentize'

  def self.colorize(*args)
    new(*args).colorize
  end

  def initialize(target, lexer = :text, format = :html, encoding = self.class.default_encoding)
    @target  = target
    @options = { :l => lexer, :f => format, :O => "encoding=#{encoding}" }
  end

  def execute(options = {})
    proc_options = {}
    proc_options[:timeout] = options.delete(:timeout) || 5
    command = convert_options(options)
    command.unshift(bin)
    Process.new(command, env={}, proc_options.merge(:input => write_target))
  end

  def colorize(options = {})
    execute(options).out
  end
  alias_method :to_s, :colorize

  def convert_options(options = {})
    @options.merge(options).inject([]) do |memo, (flag, value)|
      validate_shell_args(flag.to_s, value.to_s)
      memo << "-#{flag}" << value.to_s
    end
  end

  def write_target
    if @target.respond_to?(:read)
      out = @target.read
      @target.close
      out
    else
      @target.to_s
    end
  end

  def validate_shell_args(flag, value)
    if flag !~ /^[a-z]+$/i
      raise ShellArgumentError, "Flag is invalid: #{flag.inspect}"
    end<commit_msg>make the default timeout value configurable<commit_after>require 'albino/process'
##
# Wrapper for the Pygments command line tool, pygmentize.
#
# Pygments: http://pygments.org/
#
# Assumes pygmentize is in the path.  If not, set its location
# with Albino.bin = '/path/to/pygmentize'
#
# Use like so:
#
#   @syntaxer = Albino.new('/some/file.rb', :ruby)
#   puts @syntaxer.colorize
#
# This'll print out an HTMLized, Ruby-highlighted version
# of '/some/file.rb'.
#
# To use another formatter, pass it as the third argument:
#
#   @syntaxer = Albino.new('/some/file.rb', :ruby, :bbcode)
#   puts @syntaxer.colorize
#
# You can also use the #colorize class method:
#
#   puts Albino.colorize('/some/file.rb', :ruby)
#
# Another also: you get a #to_s, for somewhat nicer use in Rails views.
#
#   ... helper file ...
#   def highlight(text)
#     Albino.new(text, :ruby)
#   end
#
#   ... view file ...
#   <%= highlight text %>
#
# The default lexer is 'text'.  You need to specify a lexer yourself;
# because we are using STDIN there is no auto-detect.
#
# To see all lexers and formatters available, run `pygmentize -L`.
#
# <NAME> // <EMAIL>
#         GitHub // http://github.com
#
class Albino
  class ShellArgumentError < ArgumentError; end

  VERSION = '1.2.2'

  class << self
    attr_accessor :bin, :default_encoding, :timeout_threshold
  end

  self.timeout_threshold = 10
  self.default_encoding  = 'utf8'
  self.bin = 'pygmentize'

  def self.colorize(*args)
    new(*args).colorize
  end

  def initialize(target, lexer = :text, format = :html, encoding = self.class.default_encoding)
    @target  = target
    @options = { :l => lexer, :f => format, :O => "encoding=#{encoding}" }
  end

  def execute(options = {})
    proc_options = {}
    proc_options[:timeout] = options.delete(:timeout) || self.class.timeout_threshold
    command = convert_options(options)
    command.unshift(bin)
    Process.new(command, env={}, proc_options.merge(:input => write_target))
  end

  def colorize(options = {})
    execute(options).out
  end
  alias_method :to_s, :colorize

  def convert_options(options = {})
    @options.merge(options).inject([]) do |memo, (flag, value)|
      validate_shell_args(flag.to_s, value.to_s)
      memo << "-#{flag}" << value.to_s
    end
  end

  def write_target
    if @target.respond_to?(:read)
      out = @target.read
      @target.close
      out
    else
      @target.to_s
    end
  end

  def validate_shell_args(flag, value)
    if flag !~ /^[a-z]+$/i
      raise ShellArgumentError, "Flag is invalid: #{flag.inspect}"
    end
{'fraction_non_alphanumeric': 0.14158592050622898, 'fraction_numerical': 0.002175202689341507, 'mean_word_length': 3.2220367278797997, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2073802', 'n_tokens_mistral': 1797, 'n_tokens_neox': 1732, 'n_words': 466}