<commit_before>require 'ripper'
require 'pp'
require 'json'
require 'optparse'


# --------------------- utils ---------------------
def banner(s)
  puts "\033[93m#{s}:\033[0m"
end


class AstSimplifier

  def initialize(filename)
    @filename = filename

    f = File.open(filename, 'rb')
    @src = f.read
    f.close

    detected_enc = detect_encoding(@src)
    if detected_enc
      begin
        @src.force_encoding(detected_enc)
      rescue
        @src.force_encoding('utf-8')
      end
    else
      # default to UTF-8
      @src.force_encoding('utf-8')
    end

    @src.encode('utf-8',
                {:undef => :replace,
                 :invalid => :replace,
                 :universal_newline => true}
    )

    @line_starts = [0]
    find_line_starts
  end


  def detect_encoding(s)
    # take first two lines
    header = s.match('^.*\n?.*\n?')
    if header and header[0]
      matched = header[0].match('^\s*#.*coding\s*[:=]\s*([\w\d\-]+)')
      if matched and matched[1]
        matched[1]
      end
    end
  end


  # initialize the @line_starts array
  # used to convert (line,col) location to (start,end)
  def find_line_starts
    lines = @src.split(/\n/)
    total = 0
    lines.each { |line|
      total += line.length + 1 # line and \n
      @line_starts.push(total)
    }
  end


  def node_start(loc)
    line = loc[0]
    col = loc[1]
    @line_starts[line-1] + col
  end


  def ident_end(start_idx)
    idx = start_idx
    while idx < @src.length and @src[idx].match /[[:alpha:]0-9_@$]/
      idx += 1
    end
    idx
  end


  def simplify
    tree = Ripper::SexpBuilder.new(@src).parse

    if $options[:debug]
      banner 'sexp'
      pp tree
    end

    simplified = convert(tree)
    simplified, ignore1, ignore2 = convert_locations(simplified)

    if $options[:debug]
      banner 'simplified'
      pp simplified
    end

    simplified
  end


  def convert_locations(obj)
    if obj.is_a?(Hash)
      #if obj[:type] == :binary and not obj[:left]
      #  puts "problem obj: #{obj.inspect}"
      #end
      ret = {}
      whole_start = -1;
      whole_end = -1;

      obj.each do |k, v|
        if k == :location
          start_idx = node_start(v)
          end_idx = ident_end(start_idx)
          ret[:start] = start_idx
          ret[:end] = end_idx
          whole_start = start_idx
          whole_end = end_idx
        else
          new_node, start_idx, end_idx = convert_locations(v)
          ret[k] = new_node

          if start_idx > 0 && (whole_start < 0 || whole_start > start_idx)
            whole_start = start_idx
          end

          if end_idx > 0 && (whole_end < 0 || whole_end < end_idx)
            whole_end = end_idx
          end
        end
      end

      ret[:start] = whole_start
      ret[:end] = whole_end
      return ret, whole_start, whole_end

    elsif obj.is_a?(Array)
      ret = []
      whole_start = -1
      whole_end = -1

      for v in obj
        new_node, start_idx, end_idx = convert_locations(v)
        ret.push(new_node)
        if  start_idx > 0 && (whole_start < 0 || whole_start > start_idx)
          whole_start = start_idx
        end

        if end_idx > 0 && (whole_end < 0 || whole_end < end_idx)
          whole_end = end_idx
        end
      end

      return ret, whole_start, whole_end
    else
      return obj, -1, -1
    end

  end


  # ------------------- conversion --------------------
  # convert and simplify ruby's "sexp" into a hash
  # exp -> hash
  def convert(exp)
    if exp == nil
      {}
    elsif exp == false
      {
          :type => :name,
          :id => 'false',
      }
    elsif exp == true
      {
          :type => :name,
          :id => 'true',
      }
    else<commit_msg>fix corner case<commit_after>require 'ripper'
require 'pp'
require 'json'
require 'optparse'


# --------------------- utils ---------------------
def banner(s)
  puts "\033[93m#{s}:\033[0m"
end


class AstSimplifier

  def initialize(filename)
    @filename = filename

    f = File.open(filename, 'rb')
    @src = f.read
    f.close

    detected_enc = detect_encoding(@src)
    if detected_enc
      begin
        @src.force_encoding(detected_enc)
      rescue
        @src.force_encoding('utf-8')
      end
    else
      # default to UTF-8
      @src.force_encoding('utf-8')
    end

    @src.encode('utf-8',
                {:undef => :replace,
                 :invalid => :replace,
                 :universal_newline => true}
    )

    @line_starts = [0]
    find_line_starts
  end


  def detect_encoding(s)
    # take first two lines
    header = s.match('^.*\n?.*\n?')
    if header and header[0]
      matched = header[0].match('^\s*#.*coding\s*[:=]\s*([\w\d\-]+)')
      if matched and matched[1]
        matched[1]
      end
    end
  end


  # initialize the @line_starts array
  # used to convert (line,col) location to (start,end)
  def find_line_starts
    lines = @src.split(/\n/)
    total = 0
    lines.each { |line|
      total += line.length + 1 # line and \n
      @line_starts.push(total)
    }
  end


  def node_start(loc)
    line = loc[0]
    col = loc[1]
    @line_starts[line-1] + col
  end


  def ident_end(start_idx)
    idx = start_idx
    while idx < @src.length and @src[idx].match /[[:alpha:]0-9_@$]/
      idx += 1
    end
    idx
  end


  def simplify
    tree = Ripper::SexpBuilder.new(@src).parse

    if $options[:debug]
      banner 'sexp'
      pp tree
    end

    simplified = convert(tree)
    simplified = find_locations(simplified)

    if $options[:debug]
      banner 'simplified'
      pp simplified
    end

    simplified
  end


  def find_locations(obj)
    def find1(obj)
      if obj.is_a?(Hash)
        #if obj[:type] == :binary and not obj[:left]
        #  puts "problem obj: #{obj.inspect}"
        #end
        ret = {}
        whole_start = nil;
        whole_end = nil;

        obj.each do |k, v|
          if k == :location
            start_idx = node_start(v)
            end_idx = ident_end(start_idx)
            ret[:start] = start_idx
            ret[:end] = end_idx
            whole_start = start_idx
            whole_end = end_idx
          else
            new_node, start_idx, end_idx = find1(v)
            ret[k] = new_node

            if start_idx && (!whole_start || whole_start > start_idx)
              whole_start = start_idx
            end

            if end_idx && (!whole_end || whole_end < end_idx)
              whole_end = end_idx
            end
          end
        end

        if whole_start
          ret[:start] = whole_start
          ret[:end] = whole_end
        end
        return ret, whole_start, whole_end

      elsif obj.is_a?(Array)
        ret = []
        whole_start = nil
        whole_end = nil

        for v in obj
          new_node, start_idx, end_idx = find1(v)
          ret.push(new_node)
          if  start_idx && (!whole_start || whole_start > start_idx)
            whole_start = start_idx
          end

          if end_idx && (!whole_end || whole_end < end_idx)
            whole_end = end_idx
          end
        end

        return ret, whole_start, whole_end
      else
        return obj, nil, nil
      end
    end

    node, _, _ = find1(obj)
    node
  end


  # ------------------- conversion --------------------
  # convert and simplify ruby's "sexp" into a hash
  # exp -> hash
  def convert(exp)
    if exp == nil
      {}
    elsif exp == false
      {
          :type => :name,
          :id => 'false',
      }
    elsif exp == true
      {
          :type => :name,
          :id => 'true',
      }
    else
{'fraction_non_alphanumeric': 0.1328855524825497, 'fraction_numerical': 0.009482418016594232, 'mean_word_length': 1.6561734872332983, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4760156', 'n_tokens_mistral': 2606, 'n_tokens_neox': 2485, 'n_words': 612}