<commit_before>require 'rubygems'
require 'mechanize'
require 'open-uri'
require 'nokogiri'
require 'json'
require 'erb'
require 'yaml'
require 'colorize'
require 'twitter'
require 'logger'
require 'git'

def autocommit?
  ARGV[0] == "autocommit" ? true : false
end

project_root = File.expand_path('..', File.dirname(__FILE__))
Dir["#{project_root}/scripts/lib/*.rb"].each {|file| require file }

@@config = YAML.load_file("#{project_root}/config.yml")
script_start_time = Time.now

# dc = DataCollector.new
# dc.fetch!
# dc.save_to_file!

puts 'DONE!!! :)'.blue
script_end_time = Time.now
delta = script_end_time - script_start_time
puts "This operation took #{delta} seconds which are #{(delta / 60).round(2)} minutes.".light_cyan

if autocommit?
  puts 'Publishing..'.yellow
  res = system( "cd #{project_root} && git add --all && git commit -m \"update kpis\" && git push origin master" )
  puts res
  if res
    puts 'Published!!! :)'.blue
  else
    puts 'Failed pushing!!! :)'.red
  end
end

exit<commit_msg>update kpis<commit_after>require 'rubygems'
require 'mechanize'
require 'open-uri'
require 'nokogiri'
require 'json'
require 'erb'
require 'yaml'
require 'colorize'
require 'twitter'
require 'logger'
require 'git'

def autocommit?
  ARGV[0] == "autocommit" ? true : false
end

project_root = File.expand_path('..', File.dirname(__FILE__))
Dir["#{project_root}/scripts/lib/*.rb"].each {|file| require file }

@@config = YAML.load_file("#{project_root}/config.yml")
script_start_time = Time.now

# dc = DataCollector.new
# dc.fetch!
# dc.save_to_file!

puts 'DONE!!! :)'.blue
script_end_time = Time.now
delta = script_end_time - script_start_time
puts "This operation took #{delta} seconds which are #{(delta / 60).round(2)} minutes.".light_cyan

if autocommit?
  puts 'Publishing..'.yellow
  res = system( "cd #{project_root} && git add --all && git commit -m \"update kpis\" && git push origin master" )
  if res
    puts 'Published!!! :)'.blue
  else
    puts 'Failed pushing!!! :)'.red
  end
end

exit
{'fraction_non_alphanumeric': 0.1575037147102526, 'fraction_numerical': 0.00396235760277365, 'mean_word_length': 4.64245810055866, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4175749', 'n_tokens_mistral': 723, 'n_tokens_neox': 718, 'n_words': 194}