<commit_before>require_relative './lib/scraper.rb'
require_relative './lib/parser.rb'

years = (2005..2014)
# years.each_with_index do |yr, i|
#   rush = Scraper.new('rushing', yr)
#   pass = Scraper.new('passing', yr)
#   rec = Scraper.new('receiving', yr)
  
#   ## Run the methods to grab the pages
#   rush.get_html_page
#   rush.write_to_file
#   pass.get_html_page
#   pass.write_to_file
#   rec.get_html_page
#   rec.write_to_file
# end
  

years.each_with_index do |yr, i|
  rush = Parser.new('rushing', yr)
  pass = Parser.new('passing', yr)
  rec = Parser.new('receiving', yr)
## Run methods once each instance has been instantiated
  rush.open_pages
  rush.row_length
  rush.write_to_csv
  pass.open_pages
  pass.row_length
  pass.write_to_csv
  rec.open_pages
  rec.row_length
  rec.write_to_csv
end
<commit_msg>Add parser commands to init, comment some lines<commit_after>require_relative './lib/scraper.rb'
require_relative './lib/parser.rb'
require_relative './lib/csvtojson.rb'

years = (2005..2014)

# methods below are for grabbing the html pages if
# not in the html_hold directory already

# years.each_with_index do |yr, i|
#   rush = Scraper.new('rushing', yr)
#   pass = Scraper.new('passing', yr)
#   rec = Scraper.new('receiving', yr)
  
#   ## Run the methods to grab the pages
#   rush.get_html_page
#   rush.write_to_file
#   pass.get_html_page
#   pass.write_to_file
#   rec.get_html_page
#   rec.write_to_file
# end
  

# years.each_with_index do |yr, i|
#   rush = Parser.new('rushing', yr)
#   pass = Parser.new('passing', yr)
#   rec = Parser.new('receiving', yr)
# ## Run methods once each instance has been instantiated
#   rush.open_pages
#   rush.row_length
#   rush.write_to_csv
#   pass.open_pages
#   pass.row_length
#   pass.write_to_csv
#   rec.open_pages
#   rec.row_length
#   rec.write_to_csv
# end


parser = Jsonifier.new
parser.open_csv('./csv_hold/2014/passing.csv')
parser.write_json('./json_hold/2014/passing.json')
{'fraction_non_alphanumeric': 0.12614445574771108, 'fraction_numerical': 0.012207527975584944, 'mean_word_length': 3.832923832923833, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '267207', 'n_tokens_mistral': 786, 'n_tokens_neox': 758, 'n_words': 156}