<commit_before># parse_fnba.rb
require 'rubygems'
require 'nokogiri'
require 'open-uri'
require 'optparse'
require 'pp'		#pp for debugging

# Test URLs
test_local_weekly = "projections_weekly.html"
test_local_daily = "projections_daily.html"
test_remote_weekly = "http://games.espn.go.com/fba/playertable/prebuilt/manageroster?leagueId=23829&teamId=14&seasonId=2015&scoringPeriodId=1&view=stats&context=clubhouse&version=projections&ajaxPath=playertable/prebuilt/manageroster&managingIr=false&droppingPlayers=false&asLM=false"
test_remote_daily = "http://games.espn.go.com/fba/playertable/prebuilt/manageroster?leagueId=40207&teamId=12&seasonId=2015&scoringPeriodId=1&view=stats&context=clubhouse&version=projections&ajaxPath=playertable/prebuilt/manageroster&managingIr=false&droppingPlayers=false&asLM=false"

# Parse command line options
def parseOpts(args)

	# Default options
	options = {:verbose => false, :season => '2015'} 

	OptionParser.new do |opts|

		opts.banner = "Usage: predict.rb [options]"

		opts.separator ""
		opts.separator "Specific options:"

		opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
			options[:verbose] = v
		end

		# Get League 
		opts.on("-l", "--league LEAGUE_ID", "ESPN League ID") do |league|
			options[:league] = league
		end

		# Get Team ID
		opts.on("-t", "--team TEAM_ID,[TEAM2_ID]", Array,"ESPN Team ID(s)") do |team|
			options[:team] = team
		end

		# Enable Local Test Mode
		opts.on("--test [local,remote]", "Use local test file") do |test|
			options[:test] = test || "local"
		end

		# Choose format
		opts.on("--format [daily,weekly]", "Use local test file") do |format|
			options[:format] = format || "weekly"
		end

		# Display Help
		opts.on_tail('-h', '--help', 'Displays Help') do
			puts opts
		exit 0
		end

	end.parse!(args)
	return options

end # parseOpts()

# Parse FNBA team from ESPN URL
def parseTeam(url,verbose)

	# Open Team URL 
	# TODO: Add error handling
	page = Nokogiri::HTML(open(url))

	# Get team table
	table = page.css("table.playerTableTable");

	# Get players as rows
	rows = table.css("tr.pncPlayerRow")

	# Parse players from ESPN Player Table

	@players = rows.collect do |row|
		
		# Print HTML if verbose set for debugging
		if verbose
			pp row
		end

		player = {}
		[
			[:playerid, 'td[2]/a/@playerid'],
			[:slot, 'td[1]'],
			[:name, 'td[@class="playertablePlayerName"]/a'],
			[:fgPercent, 'td[@class="playertableStat "][2]'],
			[:ftPercent, 'td[@class="playertableStat "][4]'],
			[:threePointers, 'td[@class="playertableStat "][5]'],
			[:rebounds, 'td[@class="playertableStat "][6]'],
			[:assists, 'td[@class="playertableStat "][7]'],
			[:steals, 'td[@class="playertableStat "][8]'],
			[:blocks, 'td[@class="playertableStat "][9]'],
			[:turnovers, 'td[@class="playertableStat "][10]'],
			[:points, 'td[@class="playertableStat "][11]'],
			[:pr15, 'td[@class="playertableData"][1]'],
			[:ownPercent, 'td[@class="playertableData"][2]'],
			[:ownChange, 'td[@class="playertableData"][3]'],
		].each do |name, xpath|
			player[name] = row.xpath(xpath).text
		end
		
		# Special parsing for complex fields
		player[:team],*player[:position] = row.xpath('td[2]/text()').to_s.gsub('&nbsp;', ' ').delete(',').split(' ')
		# TODO: Improve opponent parsing to work for weekly and daily leagues
		player[:opponents] = row.xpath('td[4]/*/text()')

		player[:fgm],player[:fga] = row.xpath('td[@class="playertableStat "][1]').text.to_s.split('/')
		player[:ftm],player[:fta] = row.xpath('td[@class="playertableStat "][3]').text.to_s.split('/')	
		player
	end

	@players

end #parseTeam()

# Prints data for a given team
def printTeam(team)

	# Iterate through players
	team.each do |player|
		player.each {|key,value| puts "#{key},#{value}"}
	end

end #printTeam()


def compareTeams(team)

# Fields
#playerid,1001
#slot,Bench
#name,<NAME>
#fgPercent,.513
#ftPercent,.568
#threePointers,0.0
#rebounds,8.7
#assists,0.6
#steals,0.7
#blocks,1.2
#turnovers,1.5
#points,8.9
#pr15,--
#ownPercent,6.4
#ownChange,+2
#team,Pho
#position,["PF", "C"]
#opponents,LALSA@Uta
#fgm,3.9
#fga,7.7
#ftm,1.0
#fta,1.8

team.each do |player|
	player.each {|key,value| puts "#{key},#{value}"}
end




end #compareTeams()


# Parse command line options if supplied, otherwise print help
if ARGV.count > 0
	options = parseOpts(ARGV)
else
	ARGV << "--help"
	parseOpts(ARGV)
	exit 0
end

# If a league and team is supplied, use a specific URL, if test supplied, use local test URL
if options[:league] && options[:team][0]
	url = "http://games.espn.go.com/fba/playertable/prebuilt/manageroster?leagueId=#{options[:league]}&teamId=#{options[:team][0]}&seasonId=2015&scoringPeriodId=1&view=stats&context=clubhouse&version=projections&ajaxPath=playertable/prebuilt/manageroster&managingIr=false&droppingPlayers=false&asLM=false"
	if options[:team][1]
		url2 = "http://games.espn.go.com/fba/playertable/prebuilt/manageroster?leagueId=#{options[:league]}&teamId=#{options[:team][1]}&seasonId=2015&scoringPeriodId=1&view=stats&context=clubhouse&version=projections&ajaxPath=playertable/prebuilt/manageroster&managingIr=false&droppingPlayers=false&asLM=false"
	end
elsif options[:test]
	# Select test URL
	if options[:test] == "local"
		if options[:format] == "daily"
			url = "#{test_local_daily}"
		else 
			url = "#{test_local_weekly}"
		end
	else
		if options[:format] == "daily"
			url = "#{test_remote_daily}"
		else 
			url = "#{test_remote_weekly}"
		end
	end
else
	puts "Something went wrong!\n"
	exit 0
end

puts "Parsing data for URL: #{url}\n"

# Handle one or two teams
if options[:team][1]
	@team1 = parseTeam(url,options[:verbose])
	@team2 = parseTeam(url2,options[:verbose])
	compareTeams(@team1,@team2)
	#debug printing
	#printTeam(@team1)
	#printTeam(@team2)
else
	@team1 = parseTeam(url,options[:verbose])
	printTeam(@team1)
	#showTeam
end
<commit_msg>Support arbitrary number of teams, cleaned up code, broke --test, added cache flag<commit_after># parse_fnba.rb
require 'rubygems'
require 'nokogiri'
require 'open-uri'
require 'optparse'
require 'pp'		


# Parse command line options
def parseOpts(args)

	# Default options
	options = {:verbose => false, :season => '2015'} 

	OptionParser.new do |opts|

		opts.banner = "Usage: predict.rb [options]"

		opts.separator ""
		opts.separator "Specific options:"

		opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
			options[:verbose] = v
		end

		# Get League 
		opts.on("-l", "--league LEAGUE_ID", "ESPN League ID") do |league|
			options[:league] = league
		end

		# Get Team ID
		opts.on("-t", "--team TEAM_ID,[TEAM2_ID, ...]", Array,"ESPN Team ID(s)") do |team|
			options[:team] = team
		end

		# Use defaults for testing
		opts.on("--test", "Use default test values (league = 23829, team = 12,14)") do |test|
			options[:test] = test
		end

		# Cache content, use cached version if available
		opts.on("-c","--cache", "Use cached content if available") do |format|
			options[:cache] = cache
		end

		# Display Help
		opts.on_tail('-h', '--help', 'Displays Help') do
			puts opts
		exit 0
		end

	end.parse!(args)
	return options

end # parseOpts()

# Parse FNBA team from ESPN URL
def parseTeam(url,verbose)

	# Open Team URL 
	# TODO: Add error handling
	page = Nokogiri::HTML(open(url))

	# Get team table
	table = page.css("table.playerTableTable");

	# Get players as rows
	rows = table.css("tr.pncPlayerRow")

	# Parse players from ESPN Player Table

	@players = rows.collect do |row|
		
		# Print HTML if verbose set for debugging
		if verbose
			pp row
		end

		player = {}
		[
			[:playerid, 'td[2]/a/@playerid'],
			[:slot, 'td[1]'],
			[:name, 'td[@class="playertablePlayerName"]/a'],
			[:fgPercent, 'td[@class="playertableStat "][2]'],
			[:ftPercent, 'td[@class="playertableStat "][4]'],
			[:threePointers, 'td[@class="playertableStat "][5]'],
			[:rebounds, 'td[@class="playertableStat "][6]'],
			[:assists, 'td[@class="playertableStat "][7]'],
			[:steals, 'td[@class="playertableStat "][8]'],
			[:blocks, 'td[@class="playertableStat "][9]'],
			[:turnovers, 'td[@class="playertableStat "][10]'],
			[:points, 'td[@class="playertableStat "][11]'],
			[:pr15, 'td[@class="playertableData"][1]'],
			[:ownPercent, 'td[@class="playertableData"][2]'],
			[:ownChange, 'td[@class="playertableData"][3]'],
		].each do |name, xpath|
			player[name] = row.xpath(xpath).text
		end
		
		# Special parsing for complex fields
		player[:team],*player[:position] = row.xpath('td[2]/text()').to_s.gsub('&nbsp;', ' ').delete(',').split(' ')
		# TODO: Improve opponent parsing to work for weekly and daily leagues
		player[:opponents] = row.xpath('td[4]/*/text()')

		player[:fgm],player[:fga] = row.xpath('td[@class="playertableStat "][1]').text.to_s.split('/')
		player[:ftm],player[:fta] = row.xpath('td[@class="playertableStat "][3]').text.to_s.split('/')	
		player
	end

	@players

end #parseTeam()

# Prints data for a given team
def printTeam(team)

	# Iterate through players
	team.each do |player|
		player.each {|key,value| puts "#{key},#{value}"}
	end

end #printTeam()


def compareTeams(teams)

# Fields
#playerid,1001
#slot,Bench
#name,<NAME>
#fgPercent,.513
#ftPercent,.568
#threePointers,0.0
#rebounds,8.7
#assists,0.6
#steals,0.7
#blocks,1.2
#turnovers,1.5
#points,8.9
#pr15,--
#ownPercent,6.4
#ownChange,+2
#team,Pho
#position,["PF", "C"]
#opponents,LALSA@Uta
#fgm,3.9
#fga,7.7
#ftm,1.0
#fta,1.8

teams.each do |player|
	player.each {|key,value| puts "#{key},#{value}"}
end




end #compareTeams()


# Parse command line options if supplied, otherwise print help
if ARGV.count > 0
	options = parseOpts(ARGV)
else
	ARGV << "--help"
	parseOpts(ARGV)
	exit 0
end

# Process X number of teams

if !options[:league]
	puts "\nMust supply LEAGUE_ID!\n\n"
	exit 0
end

teams = Array.new

options[:team].each do |teamId|

	url = "http://games.espn.go.com/fba/playertable/prebuilt/manageroster?leagueId=#{options[:league]}&teamId=#{teamId}&seasonId=2015&scoringPeriodId=1&view=stats&context=clubhouse&version=projections&ajaxPath=playertable/prebuilt/manageroster&managingIr=false&droppingPlayers=false&asLM=false"
	teams << parseTeam(url,options[:verbose])
	
end

{'fraction_non_alphanumeric': 0.18041695484635906, 'fraction_numerical': 0.019662088726410433, 'mean_word_length': 5.1308298001211385, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '174258', 'n_tokens_mistral': 4135, 'n_tokens_neox': 3749, 'n_words': 799}