<commit_before>require 'rubygems'
require 'nokogiri'
require 'open-uri'
require 'fileutils'

BASE_URL = 'http://espn.go.com/nfl/statistics/player/_/stat/'
BASE_DIR = {:passing => 'passing/sort/passingYards/seasontype/2/qualified/false/count/',
            :rushing => 'rushing/seasontype/2/qualified/false/count/',
            :receiving => 'receiving/sort/receivingYards/seasontype/2/qualified/false/count/',
            # :kicking => 'kicking/sort/fieldGoalsMade/qualified/false/count/'
          }

def scrape_passing
  counts = [1, 41, 81, 121, 161, 201, 241, 281, 321, 361, 401, 441, 481, 521, 561, 601]
  page = Nokogiri::HTML(open(BASE_URL + BASE_DIR[:passing] + '1'))
  #### Writes the first file ####
  File.open("stat_pages/passing1.html", 'w'){|f| f.write(page.to_html)}
  puts "First passing file saved successfully"

  #### Find the number of results(players) for the given stat request ####
  results_div = page.css('#my-players-table div.totalResults').first.content
  number_results_arr = results_div.split(' ')
  number_results = number_results_arr[0].to_i

  #### Finding the correct number of pages to find ####
  if number_results % 40 == 0
    num_pages = number_results/40
  else
    num_pages = (number_results/40) + 1
  end

  puts "#{num_pages - 1} more pages need to be collected"<commit_msg>Remove kicking URL. Never used<commit_after>require 'rubygems'
require 'nokogiri'
require 'open-uri'
require 'fileutils'

BASE_URL = 'http://espn.go.com/nfl/statistics/player/_/stat/'
BASE_DIR = {:passing => 'passing/sort/passingYards/seasontype/2/qualified/false/count/',
            :rushing => 'rushing/seasontype/2/qualified/false/count/',
            :receiving => 'receiving/sort/receivingYards/seasontype/2/qualified/false/count/',
            # :kicking => 'kicking/sort/fieldGoalsMade/qualified/false/count/'  ##Never implemented
          }

def scrape_passing
  counts = [1, 41, 81, 121, 161, 201, 241, 281, 321, 361, 401, 441, 481, 521, 561, 601]
  page = Nokogiri::HTML(open(BASE_URL + BASE_DIR[:passing] + '1'))
  #### Writes the first file ####
  File.open("stat_pages/passing1.html", 'w'){|f| f.write(page.to_html)}
  puts "First passing file saved successfully"

  #### Find the number of results(players) for the given stat request ####
  results_div = page.css('#my-players-table div.totalResults').first.content
  number_results_arr = results_div.split(' ')
  number_results = number_results_arr[0].to_i

  #### Finding the correct number of pages to find ####
  if number_results % 40 == 0
    num_pages = number_results/40
  else
    num_pages = (number_results/40) + 1
  end

  puts "#{num_pages - 1} more pages need to be collected"
{'fraction_non_alphanumeric': 0.14846671652954377, 'fraction_numerical': 0.044128646222887064, 'mean_word_length': 3.890310786106033, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2513196', 'n_tokens_mistral': 1036, 'n_tokens_neox': 907, 'n_words': 205}