<jupyter_start><jupyter_text>Source: NCAA<jupyter_code>#hide
import core_constants as cc
import functions as fx
import json
import pandas as pd
import sqlite3 as sql<jupyter_output><empty_output><jupyter_text>Set Notebook Settings<jupyter_code>#Load NCAA Site Schema
schoolsList = json.loads(open('..//config//schools.json', "r").read())
ncaaDates = json.loads(open('..//config//ncaa_dates.json', "r").read())<jupyter_output><empty_output><jupyter_text>Get & Save the Teams Page HTML Source: https://stats.ncaa.org/team/392/roster/15280> This page contains a roster for any given team in any year (using their own ids)<jupyter_code>fx.get_NCAA(schoolsList, ncaaDates)<jupyter_output><empty_output><jupyter_text>Process Local NCAA HTML Files> All of this processing is done locally, using the files saved in the previous few steps.  This creates an exhaustive store of all the fields grabbed from the scrapes.<jupyter_code>conferences = cc.get_availableConferences()

cc.save_records('scrapedData', 'ncaa', fx.process_NCAA(conferences))<jupyter_output><empty_output><jupyter_text>Save to DB<jupyter_code>fx.toDB_NCAA()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11159546643417612, 'fraction_numerical': 0.007846556233653008, 'mean_word_length': 7.631578947368421, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '355060', 'license': 'Apache-2.0', 'n_tokens_mistral': 407, 'n_tokens_neox': 390, 'n_words': 94, 'path': 'j_notebooks/NCAA.ipynb', 'repo_name': 'andrewt-cville/fb_recruiting', 'size': '1147'}