<jupyter_start><jupyter_text>IntroThis notebook was created to aggregate NCAA Men's Basketball rankings at a given time, across 3 of the most well regarding ranking systems/sites to date.The notebook scrapes ranking data from the www.ncaa.com (NET), www.kenpom.com (KenPom) and www.barttorvik.com (BartTorvik).The notebook makes use of a few .csv files used to standardize the naming convention for all 353 D1 Men's Basketball Programs. These .csv files are used to "convert" to a standard naming convention - going off of the naming convention used by the NCAA and NET. The notebook utilizes BeautifulSoup and Requests to scrape web data.The notebook pulls only the overall rank from the respective sites - though the code could be modified to pull further statistics for each team on each site.<jupyter_code>#https://www.ncaa.com/rankings/basketball-men/d1/ncaa-mens-basketball-net-rankings 
#https://kenpom.com/
#http://www.barttorvik.com/#<jupyter_output><empty_output><jupyter_text>NET<jupyter_code>import pandas
import requests
import bs4


url = 'https://www.ncaa.com/rankings/basketball-men/d1/ncaa-mens-basketball-net-rankings'
header = {'user-agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Safari/605.1.15'}#trick Amazon 
page = requests.get(url, headers=header)
soup = bs4.BeautifulSoup(page.text, 'html5lib')
page.reason
tags1 = soup.find_all('tr')
data = [(int(t.contents[1].text), t.contents[5].text) for t in tags1[1:]]
pandas.options.display.max_rows = 4000
NETtable = pandas.DataFrame(data, columns = ['Rank', 'School'])
NETtable<jupyter_output><empty_output><jupyter_text>KenPom<jupyter_code>url = 'https://kenpom.com/'
header = {'user-agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Safari/605.1.15'}#trick Amazon 
page = requests.get(url, headers=header)
soup = bs4.BeautifulSoup(page.text, 'html5lib')
page.reason
trs = soup.find_all('tr')
trs

tds =[tr.find_all('td') for tr in trs]

for x in tds:
    if x == []:
        tds.remove(x)
        
    if x == []:
        tds.remove(x)
newtd= tds[2:]
dataKP = [(int(t[0].string), t[1].a.string) for t in tds]
KPtable = pandas.DataFrame(dataKP, columns = ['Rank', 'School'])
convert = pandas.read_csv('NETtoKP.csv')
convert_dict = convert.set_index('KP').T.to_dict('records')[0]
KPtable = KPtable.replace({'School': convert_dict})
KPtable<jupyter_output><empty_output><jupyter_text><NAME><jupyter_code>url = 'http://www.barttorvik.com/#'
header = {'user-agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Safari/605.1.15'}#trick Amazon 
page = requests.get(url, headers=header)
soup = bs4.BeautifulSoup(page.text, 'html5lib')
page.reason
tags2 = soup.find_all('tr',{'class':"seedrow"})
dataBT = [(int(tags2[t].contents[1].text), tags2[t].contents[3].text.split(u'\xa0')[0]) for t in range(len(tags2))]
BTtable = pandas.DataFrame(dataBT, columns = ['Rank', 'School'])
convert2 = pandas.read_csv('NETtoBT.csv')
convert_dict2 = convert2.set_index('BT').T.to_dict('records')[0]
BTtable = BTtable.replace({'School': convert_dict2})
BTtable<jupyter_output><empty_output><jupyter_text>Build final merged DataFrame with NET, KenPom and BartTorvik Rankings<jupyter_code>from functools import reduce

dfs = [NETtable, KPtable, BTtable]

df_final = reduce(lambda left,right: pandas.merge(left,right,on='School'), dfs)
rankings = df_final.rename(columns={"Rank_x": "NET", "Rank_y": "KenPom", "Rank": "BartTorvik"})
rankings = rankings[['School', 'NET', 'KenPom', 'BartTorvik']]
rankings<jupyter_output><empty_output><jupyter_text>Save locally as .csv<jupyter_code>from datetime import datetime
current_time = datetime.now()
date = (current_time.strftime('%m-%d-%Y'))

save_name = 'CBB_Rankings_' + date + '.csv'
rankings.to_csv(save_name)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14866939611054247, 'fraction_numerical': 0.026356192425793245, 'mean_word_length': 5.821989528795812, 'pattern_counts': {'":': 3, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 4, 'lorem ipsum': 0, 'www.': 7, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '596958', 'license': 'MIT', 'n_tokens_mistral': 1522, 'n_tokens_neox': 1408, 'n_words': 328, 'path': 'College Basketball Rankings (NET, KenPom, BartTorvik).ipynb', 'repo_name': 'dylorr/CBB-Rankings', 'size': '3913'}