<commit_before>require 'pygments.rb'

module Diff
  def main(argv=nil)
    output = ''
    infos = files_infos(argv)
    infos.each do |file|
      output << parse_diff(file)
    end
    output
  end

  def files_infos(argv)
    files = []
    names = argv == [] ? files_names : argv
    names.map do |name|
      files << { name: name, diff: file_diff(name) }
    end
    files
  end

  def parse_diff(file_info)
    extension = file_extension(file_info[:name])
    splited = split_diff(file_info[:diff])
    heads, codes = splited[:heads], splited[:codes]
    complete_file_diff = ''

    (0...heads.size).each do |i|
      head = heads[i]
      head = prepare_head_for_output(head) if i > 0 #TODO: wrong!
      parsed_head = parse_with_diff(prepare_head_for_output(head))
      parsed_code = parse_with_lang(codes[i], extension)
      parsed_code = parse_with_diff(parsed_code)
      complete_file_diff << (parsed_head + parsed_code)
    end

    complete_file_diff
  end

  def prepare_head_for_output(head)
    head.insert 0, "\n"
  end

  def file_extension(file_name)
    extension = file_name.gsub /(.+\.)/, ''
    if extension.empty? || file_name == extension
      extension = files_with_no_extension[file_name]
      extension ||= 'text'
    end
    extension.to_sym
  end

  def split_diff(diff)
    { heads: split_heads(diff),
      codes: split_codes(diff) }
  end

  def split_codes(diff)
    codes = diff.split(/^@@ .* @@/)
    codes[1..codes.size]
  end

  def split_heads(diff)
    heads = diff.scan(/diff(?:.*\n){4}@@ .* @@/)
    heads << diff.scan(/^@@ .* @@/).tap(&:shift)
    heads.flatten
  end

  def file_diff(file)
    `git diff #{file}`
  end

  def files_names
    git_status.scan(/modified: .*/).map { |n| n.gsub(/modified: */, '') }
  end

  def git_status
    `git status`
  end

  def parse_with_lang(code, lang)
    process(code, lang)<commit_msg>fix error: no newline before heads<commit_after>require 'pygments.rb'

module Diff
  def main(argv=nil)
    output = ''
    infos = files_infos(argv)
    infos.each do |file|
      output << parse_diff(file)
    end
    output
  end

  def files_infos(argv)
    files = []
    names = argv == [] ? files_names : argv
    names.map do |name|
      files << { name: name, diff: file_diff(name) }
    end
    files
  end

  def parse_diff(file_info)
    extension = file_extension(file_info[:name])
    splited = split_diff(file_info[:diff])
    heads, codes = splited[:heads], splited[:codes]
    complete_file_diff = ''

    (0...heads.size).each do |i|
      parsed_head = parse_with_diff(heads[i])
      parsed_code = parse_with_lang(codes[i], extension)
      parsed_code = parse_with_diff(parsed_code)
      complete_file_diff << (parsed_head + parsed_code + "\n")
    end

    complete_file_diff
  end

  def file_extension(file_name)
    extension = file_name.gsub /(.+\.)/, ''
    if extension.empty? || file_name == extension
      extension = files_with_no_extension[file_name]
      extension ||= 'text'
    end
    extension.to_sym
  end

  def split_diff(diff)
    { heads: split_heads(diff),
      codes: split_codes(diff) }
  end

  def split_codes(diff)
    codes = diff.split(/^@@ .* @@/)
    codes[1..codes.size]
  end

  def split_heads(diff)
    heads = diff.scan(/^diff --git(?:.*\n){4}@@ .* @@/)
    heads << diff.scan(/^@@ .* @@/).tap(&:shift)
    heads.flatten
  end

  def file_diff(file)
    `git diff #{file}`
  end

  def files_names
    git_status.scan(/modified: .*/).map { |n| n.gsub(/modified: */, '') }
  end

  def git_status
    `git status`
  end

  def parse_with_lang(code, lang)
    process(code, lang)
{'fraction_non_alphanumeric': 0.1406465874550981, 'fraction_numerical': 0.0022105554020447637, 'mean_word_length': 2.863393810032017, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3338721', 'n_tokens_mistral': 1309, 'n_tokens_neox': 1282, 'n_words': 265}