<jupyter_start><jupyter_text>Loading Data from API<jupyter_code>import http.client
import pandas as pd
import json
from pandas.io.json import json_normalize
import time

conn = http.client.HTTPSConnection("api.sportradar.us")<jupyter_output><empty_output><jupyter_text>Massive Loop<jupyter_code>GamesAway = {
    "Pirates1" : "/mlb/trial/v6.5/en/games/f5688e9e-257b-4a26-988b-7b268920568a/pbp.json?api_key=<KEY>",
    "Pirates2" : "/mlb/trial/v6.5/en/games/9ca520f9-d6a6-43ac-895a-37d174fef1cb/pbp.json?api_key=<KEY>",
    "Pirates3" : "/mlb/trial/v6.5/en/games/801e73cc-879b-4b08-ab9b-52328df476b5/pbp.json?api_key=<KEY>",
    "Cubs1" : "/mlb/trial/v6.5/en/games/9f558c81-10db-4c1f-abe7-a11d74127cf2/pbp.json?api_key=<KEY>" ,
    "Cubs2" : "/mlb/trial/v6.5/en/games/1b425fa2-6cc0-4da5-bfd9-d4c63e97ca1c/pbp.json?api_key=<KEY>",
    "Reds1" : "/mlb/trial/v6.5/en/games/97043ab5-588b-4c7b-84d9-96f7fd31d22c/pbp.json?api_key=<KEY>",
    "Reds2" : "/mlb/trial/v6.5/en/games/be13181b-0663-48de-b493-26a4c96bea08/pbp.json?api_key=<KEY>",
    "Reds3" : "/mlb/trial/v6.5/en/games/c1ac3e5e-9ca7-4eb4-988b-a3be9d8e5e3d/pbp.json?api_key=<KEY>",
    "Reds4" : "/mlb/trial/v6.5/en/games/06e3c990-46d4-4b44-93cb-f02bd923b1b1/pbp.json?api_key=<KEY>",  
    "Brewers1" : "/mlb/trial/v6.5/en/games/4111bbd9-f767-42c1-a46b-154fe67b1b21/pbp.json?api_key=<KEY>",
    "Brewers2" : "/mlb/trial/v6.5/en/games/1e1491e8-6216-472f-aefe-c5023d316966/pbp.json?api_key=<KEY>",
    "Brewers3" : "/mlb/trial/v6.5/en/games/8ec784bc-fe9f-40ca-8e43-2de0ddaa1e95/pbp.json?api_key=<KEY>",
    "Mets1" : "/mlb/trial/v6.5/en/games/93788d5b-fdb7-44f0-b447-a9b21b7bfbf7/pbp.json?api_key=<KEY>",
    "Mets2" : "/mlb/trial/v6.5/en/games/96f27489-9017-4746-9516-8e794ab4cf38/pbp.json?api_key=<KEY>",
    "Mets3" : "/mlb/trial/v6.5/en/games/596cd5ab-b9d3-4d32-b5be-0335a235d18a/pbp.json?api_key=<KEY>"}

    GamesHome = {
    "Twins1" : "/mlb/trial/v6.5/en/games/66bb1485-dcc2-4963-a852-d68106f713f3/pbp.json?api_key=<KEY>", 
    "Twins2" : "/mlb/trial/v6.5/en/games/1726a2b9-b493-41a1-89e6-c1e839d1174c/pbp.json?api_key=<KEY>", 
    "Cubs3" : "/mlb/trial/v6.5/en/games/e070f19f-5787-4bc0-b4d3-8f0c7e9a4f8a/pbp.json?api_key=<KEY>", 
    "Cubs4" : "/mlb/trial/v6.5/en/games/e94d263b-d9bf-4521-b15b-32ac3933acc8/pbp.json?api_key=<KEY>",
    "Cubs5" : "/mlb/trial/v6.5/en/games/cea131ab-69b5-4461-a060-c5c0f0048648/pbp.json?api_key=<KEY>",
    "WhiteSox1" : "/mlb/trial/v6.5/en/games/722c8fe9-117c-4908-bde5-8d97b37197f1/pbp.json?api_key=<KEY>",
    "WhiteSox2" : "/mlb/trial/v6.5/en/games/81b010cc-0747-4c47-b90a-3810413de3ec/pbp.json?api_key=<KEY>",
    "Mets4" : "/mlb/trial/v6.5/en/games/456c97f7-9eee-4913-a0e6-2f3923782278/pbp.json?api_key=<KEY>",
    "Mets5" : "/mlb/trial/v6.5/en/games/385fc8fa-32b1-4700-bc04-d52f559ba15d/pbp.json?api_key=<KEY>",
    "Mets6" : "/mlb/trial/v6.5/en/games/42a43856-5370-4e6f-a1a2-405b4ebae1b3/pbp.json?api_key=<KEY>",
    "Reds5" : "/mlb/trial/v6.5/en/games/65584b46-6aba-4bb2-8a9e-0076146c198c/pbp.json?api_key=<KEY>",
    "Reds6" : "/mlb/trial/v6.5/en/games/f6a3ddde-280d-4be9-af87-63313cce4680/pbp.json?api_key=<KEY>",
    "Reds7" : "/mlb/trial/v6.5/en/games/145ca23f-57bb-4ee9-aec0-c5178406402e/pbp.json?api_key=<KEY>",
    "Brewers4" : "/mlb/trial/v6.5/en/games/54f4c95f-2a31-4958-8a5c-774c934578ed/pbp.json?api_key=<KEY>",
    "Brewers5" : "/mlb/trial/v6.5/en/games/dfb9ad8e-2705-4115-b4b5-36550d8ff939/pbp.json?api_key=<KEY>",
    "Brewers6" : "/mlb/trial/v6.5/en/games/5c096355-95dc-4baf-8cd8-fc3750072a42/pbp.json?api_key=<KEY>", 
    "Diamondbacks1" : "/mlb/trial/v6.5/en/games/472e8cd0-3e96-41c6-b0d5-ff3313bc089a/pbp.json?api_key=<KEY>",
    "Diamondbacks2" : "/mlb/trial/v6.5/en/games/b0c22a62-3f28-42c2-b1c9-7a7df1585b36/pbp.json?api_key=<KEY>",
    "Diamondbacks3" : "/mlb/trial/v6.5/en/games/dfb947a2-8355-4726-bca4-9659dd02c81a/pbp.json?api_key=<KEY>"}

##HOME GAME! CARDS HIT SECOND. 

def homegame(parsed):
    
    for inning in range(1,10):
        if 'game' in parsed.keys():
            level1 = parsed['game']['innings'][inning]
            
            for halfs in range(1,2):
                level2 = level1['halfs'][halfs]['events']
                  
                for i in range(0,len(level2)):
                    level3= level2[i]
                    if 'at_bat' in level3.keys():
                        
                        for x in level3['at_bat']['events']:
                            x['innings'] = inning
                                
                            if 'runners' in x.keys():
                                for r in x['runners']:

                                    if r['starting_base'] == 1:
                                        x['OnFirst'] = 1
                                    if r['starting_base'] == 2:
                                        x['OnSecond'] = 1
                                    if r['starting_base'] == 3:
                                        x['OnThird'] = 1 
                            
                            if 'weather' in parsed['game']['innings'][4]['halfs'][0].keys():
                                    x['temperature'] = (parsed['game']['innings'][4]['halfs'][0]['weather']['current_conditions']['temp_f'])
                                    x['weather'] = (parsed['game']['innings'][4]['halfs'][0]['weather']['current_conditions']['wind'])
                                      
                            x['OppTeam'] = (parsed['game']['scoring']['away']['abbr'])
                            x['Home?'] = 1
        
                                           
                            allpitches.append(x)
## AWAY GAME! CARDS HIT FIRST. 

def awaygame(parsed):
    
    for inning in range(1,10):
        if 'game' in parsed.keys():
            level1 = parsed['game']['innings'][inning]
            
            for halfs in range(0,1):
                level2 = level1['halfs'][halfs]['events']
                  
                for i in range(0,len(level2)):
                    level3= level2[i]
                    if 'at_bat' in level3.keys():
                        
                        for x in level3['at_bat']['events']:
                            x['innings'] = inning
                                
                            if 'runners' in x.keys():
                                for r in x['runners']:

                                    if r['starting_base'] == 1:
                                        x['OnFirst'] = 1
                                    if r['starting_base'] == 2:
                                        x['OnSecond'] = 1
                                    if r['starting_base'] == 3:
                                        x['OnThird'] = 1 
                                        
                            if 'weather' in parsed['game']['innings'][4]['halfs'][0].keys():
                                x['temperature'] = (parsed['game']['innings'][4]['halfs'][0]['weather']['current_conditions']['temp_f'])
                                x['weather'] = (parsed['game']['innings'][4]['halfs'][0]['weather']['current_conditions']['wind'])
                             
                            x['OppTeam'] = (parsed['game']['scoring']['home']['abbr'])
                            x['Home?'] = 0
                            
                            allpitches.append(x)
allpitches = []

def GettinTheGame ():
    
    for key,value in GamesHome.items():
        conn.request("GET", value)
        res = conn.getresponse()
        data = res.read()
        jason_parsed = json.loads(data)
        
        homegame(jason_parsed)
        time.sleep(1)
        
        print(key)
        
    for key,value in GamesAway.items():
        conn.request("GET", value)
        res = conn.getresponse()
        data = res.read()
        jason_parsed = json.loads(data)
        
        awaygame(jason_parsed)
        time.sleep(1)
        
        print(key)


GettinTheGame()
allseason = json_normalize(allpitches)
allseason
allseason['Home?'].unique()<jupyter_output><empty_output><jupyter_text>Cleaning ** Filling NaNs **<jupyter_code>allseason.fillna(value = 0, inplace=True)<jupyter_output><empty_output><jupyter_text>** Getting rid of nonsense entries ** - Where the pitcher/ hitter = 0- Where the pitch type = 0<jupyter_code>#drops = []

#for l in allseason.index.get_loc((allseason.loc[allseason['hitter.id'] == 0])):
 #   drops.append(l)
allseason.loc[allseason['pitcher.id'] == 0]['pitcher.id']
#allseasonclean = allseason.drop(allgames.index[[217,308,309,84,217,223,247,256,257,177,185,284,260,261,429]])
#allseasonclean = allseasonclean.drop(allseasonclean[allseasonclean['pitcher.id'] == 0].index)
allseasonclean = allseason
allseasonclean2 = allseasonclean[allseasonclean['pitcher.id'] != 0]
allseasonclean2 = allseasonclean2[allseasonclean2['pitcher.pitch_zone'] != 0]
allseasonclean2 = allseasonclean2[allseasonclean2['pitcher.pitch_type'] != 0]
allseasonclean2.columns
allseasonclean2['hitter.last_name'].value_counts()
#allseasonclean2['hitter.first_name'].value_counts()
allseasonclean2 = allseasonclean2.set_index('hitter.last_name')
allseasonclean3 = allseasonclean2.drop(labels = ['Gant','Mayers','Kelly','Flaherty','Wainwright',"O'Neill"])
allseasonclean3 = allseasonclean3.reset_index()
allseasonclean3['hitter.last_name'].value_counts()
allseasonclean3
allseasonclean3.to_csv('AllSeason')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16930430164630908, 'fraction_numerical': 0.09877854487519915, 'mean_word_length': 2.199456337071016, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '184002', 'license': 'Apache-2.0', 'n_tokens_mistral': 4176, 'n_tokens_neox': 3690, 'n_words': 334, 'path': 'Baseball Data - Pitch Recommendation/.ipynb_checkpoints/PullingMoreData-checkpoint.ipynb', 'repo_name': 'vszuflita/Metis', 'size': '10061'}