<commit_before>require 'nokogiri'
require 'open-uri'
require 'yaml'

module SalaryScraper
  # Class Pulls and Manipulates data from basketball-reference.com
  class BasketballReference
    WEBSITE = 'http://www.basketball-reference.com/contracts'
    PAYROLL = "//div[@id='div_payroll']//tbody//tr[@class='']//td"
    HEAD = "//div[@id='div_payroll']//thead//tr[@class='']//th"
<<<<<<< HEAD
    
=======
>>>>>>> d62b0730bcb45ce356cbf88cd4f7e3c32fd14924
    def get_team(team)
      doc = get_page(team)
      doc.xpath(PAYROLL)
    end

    def get_page(team)
      url = "#{WEBSITE}/#{team}.html"
      Nokogiri::HTML(open(url))
    end

    def just_text(data)
      textes_array = []
      data.each { |t| textes_array << t.text }
      textes_array
    end

    def get_column_heads(team)
      doc = get_page(team)
      doc.xpath(HEAD)
    end

    def head_array(team)
      @head_array = []
      head = get_column_heads(team)
      head.each { |x| @head_array << x.text }
      @head_array
    end

    def players_data_array(team)
      @players_data_array = []
      players_data = get_team(team)
      players_data.each { |x| @players_data_array << x.text }
      @players_data_array
    end

    def to_array_of_hashes(team)
      players_array = players_data_array(team)
      header_array = head_array(team)
      assign_key_to_value(number_arrays, players_array, header_array)
    end

    def assign_key_to_value(number_arrays, players_data_array, head_array)
      data = []
      number_arrays.times do
        piece_of_data = Hash.new
        head_array.each do |x|
          piece_of_data[x] = players_data_array.first
          players_data_array.shift
        end
        data << piece_of_data
      end
      data
    end

    def number_arrays
      @players_data_array.size / @head_array.size
    end

    def to_yaml(team)
      data = to_array_of_hashes(team)
      data.to_yaml
    end

    def check_if_team_exist(team)
      if head_array(team).size == 0 || players_data_array(team).size == 0
        return false
      else
        return true
      end
    end
  end
end<commit_msg>Added team abbreviation array<commit_after>require 'nokogiri'
require 'open-uri'
require 'yaml'

module SalaryScraper
  # Class Pulls and Manipulates data from basketball-reference.com
  class BasketballReference
    WEBSITE = 'http://www.basketball-reference.com/contracts'
    PAYROLL = "//div[@id='div_payroll']//tbody//tr[@class='']//td"
    HEAD = "//div[@id='div_payroll']//thead//tr[@class='']//th"
    ALL_TEAMS = Array['PHO', 'MIA', 'ATL', 'BOS', 'BRK', 'CHI',
                      'CLE', 'DAL', 'DEN', 'GSW', 'DET', 'LAL',
                      'LAC', 'HOU', 'IND', 'MIN', 'MEM', 'MIL',
                      'NOP', 'NYK', 'OKC', 'SAC', 'WAS', 'UTA',
                      'SAS', 'TOR', 'PHI', 'POR', 'ORL', 'CHO']
    def get_team(team)
      doc = get_page(team)
      doc.xpath(PAYROLL)
    end

    def get_page(team)
      url = "#{WEBSITE}/#{team}.html"
      Nokogiri::HTML(open(url))
    end

    def just_text(data)
      textes_array = []
      data.each { |t| textes_array << t.text }
      textes_array
    end

    def get_column_heads(team)
      doc = get_page(team)
      doc.xpath(HEAD)
    end

    def head_array(team)
      @head_array = []
      head = get_column_heads(team)
      head.each { |x| @head_array << x.text }
      @head_array
    end

    def players_data_array(team)
      @players_data_array = []
      players_data = get_team(team)
      players_data.each { |x| @players_data_array << x.text }
      @players_data_array
    end

    def to_array_of_hashes(team)
      players_array = players_data_array(team)
      header_array = head_array(team)
      assign_key_to_value(number_arrays, players_array, header_array)
    end

    def assign_key_to_value(number_arrays, players_data_array, head_array)
      data = []
      number_arrays.times do
        piece_of_data = Hash.new
        head_array.each do |x|
          piece_of_data[x] = players_data_array.first
          players_data_array.shift
        end
        data << piece_of_data
      end
      data
    end

    def number_arrays
      @players_data_array.size / @head_array.size
    end

    def to_yaml(team)
      data = to_array_of_hashes(team)
      data.to_yaml
    end

    def check_if_team_exist(team)
      if ALL_TEAMS.include?(team)
        return true
      else
        return false
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.11884969669737137, 'fraction_numerical': 0.0056167153448663226, 'mean_word_length': 2.4699922057677317, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4641609', 'n_tokens_mistral': 1624, 'n_tokens_neox': 1570, 'n_words': 292}