<commit_before>#import copper
import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup
from datetime import datetime, date
#copper.project.path = '../..'

#games = copper.read_csv('games.csv').set_index('id')
games = pd.read_csv('/Users/daniel.george/Desktop/Github/espn-analytics/nfl/games.csv')

BASE_URL = 'http://espn.go.com/nfl/boxscore?gameId={0}'

#http://espn.go.com/nba/boxscore?gameId=400277727
#http://espn.go.com/nfl/boxscore?gameId=400749514

# @Author <NAME>
# @Works Cited: <NAME>, https://github.com/danielfrg/nba

for index, row in games.iterrows():
	id = row['id']
	r = requests.get(BASE_URL.format(id))
	soup = BeautifulSoup(r.text, "html.parser")
	table = soup.find('table', {'class':'mod-data'})
	print table

# find('table', class_='mod-data')

# heads = table.find_all('thead')
# headers = heads[0].find_all('tr')[1].find_all('th')[1:]
# headers = [th.text for th in headers]
# columns = ['id', 'team', 'player'] + headers
# 
# players = pd.DataFrame(columns=columns)
# 
# def get_players(players, team_name):
#     array = np.zeros((len(players), len(headers)+1), dtype=object)
#     array[:] = np.nan
#     for i, player in enumerate(players):
#         cols = player.find_all('td')
#         array[i, 0] = cols[0].text.split(',')[0]
#         for j in range(1, len(headers) + 1):
#             if not cols[1].text.startswith('DNP'):
#                 array[i, j] = cols[j].text<commit_msg>successful pulling passing stats<commit_after>#import copper
import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup
from datetime import datetime, date
#copper.project.path = '../..'

#games = copper.read_csv('games.csv').set_index('id')
games = pd.read_csv('/Users/daniel.george/Desktop/Github/espn-analytics/nfl/games.csv')

BASE_URL = 'http://espn.go.com/nfl/boxscore?gameId={0}'

#http://espn.go.com/nba/boxscore?gameId=400277727
#http://espn.go.com/nfl/boxscore?gameId=400749514

# @Author <NAME>
# @Works Cited: <NAME>, https://github.com/danielfrg/nba

players = []

for index, row in games.iterrows():
	id = row['id']
	r = requests.get(BASE_URL.format(id))
	soup = BeautifulSoup(r.text, "html.parser")
	table = soup.find_all('table', {'class':'mod-data'})

	for row in table:
		if "Passing" in row.caption.text:
			heads = row.find_all('thead')
			for line in heads:
				headers = line.find_all('th')
				headers = [th.text for th in headers]
				columns = ['id', 'team', 'player'] + headers
				players = pd.DataFrame(columns=columns)
				
# 			team_1 = row.caption.text
# 			team_1_players = row.tbody.a.text
			array = []
			cols = row.find_all('td')
			for j in range(1, 9):
				if not cols[1].text.startswith('DNP'):
					print cols[j].text
			
# 			players = players.append(team_1)
# 			players = players.append(team_1_players)
# 			print team_1
# 			print team_1_players
			#print team_1_players_
			print array			
				
				
# def get_players(players, team_name):
#     array = np.zeros((len(players), len(headers)+1), dtype=object)
#     array[:] = np.nan
#     for i, player in enumerate(players):
#         cols = player.find_all('td')
#         array[i, 0] = cols[0].text.split(',')[0]
#         for j in range(1, len(headers) + 1):
#             if not cols[1].text.startswith('DNP'):
#                 array[i, j] = cols[j].text
{'fraction_non_alphanumeric': 0.17519566526189043, 'fraction_numerical': 0.0201685731487056, 'mean_word_length': 3.6410614525139664, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '67200', 'n_tokens_mistral': 1358, 'n_tokens_neox': 1250, 'n_words': 246}