<jupyter_start><jupyter_text>Fetch Data Scraper __Purpose is to make an API and program to fetch data from Kaggle__<jupyter_code>NUM_TEAMS = 1
EPISODES = 300
MIN_RATING = 1000<jupyter_output><empty_output><jupyter_text>Initialize<jupyter_code>test = ['a', '1', '2']
test.pop(0)
test
# !pip install kaggle-environments --upgrade
import pandas as pd
import numpy as np
import os
import requests
import json
from zipfile import ZipFile
import datetime
import time
import matplotlib.pyplot as plt
plt.rcParams['figure.figsize'] = (10, 4)
base_url = "https://www.kaggle.com/requests/EpisodeService/"
get_url = base_url + "GetEpisodeReplay"
list_url = base_url + "ListEpisodes"<jupyter_output><empty_output><jupyter_text>Get Initial Team List<jupyter_code>def get_teams(team_id):
    r = requests.post(list_url, json = {"teamId":  5118174})
    rj = r.json()
    teams_df = pd.DataFrame(rj['result']['teams'])
    teams_df.sort_values('publicLeaderboardRank', inplace = True)
    return teams_df
TEAM_ID = 5118174
df = get_teams(TEAM_ID)
df.shape
df.head()
def getTeamEpisodes(team_id):
    # request
    r = requests.post(list_url, json = {"teamId":  int(team_id)})
    rj = r.json()

    # update teams list
    global teams_df
    teams_df_new = pd.DataFrame(rj['result']['teams'])
    
    if len(teams_df.columns) == len(teams_df_new.columns) and (teams_df.columns == teams_df_new.columns).all():
        teams_df = pd.concat( (teams_df, teams_df_new.loc[[c for c in teams_df_new.index if c not in teams_df.index]] ) )
        teams_df.sort_values('publicLeaderboardRank', inplace = True)
#         print('{} teams on file'.format(len(teams_df)))
    else:
        print('teams dataframe did not match')
    
    # make df
    team_episodes = pd.DataFrame(rj['result']['episodes'])
    team_episodes['avg_score'] = -1;
    
    for i in range(len(team_episodes)):
        agents = team_episodes['agents'].loc[i]
        agent_scores = [a['updatedScore'] for a in agents if a['updatedScore'] is not None]
        if len(agent_scores) > 0:
            team_episodes.loc[i, 'avg_score'] = np.mean(agent_scores)

    team_episodes.sort_values('avg_score', ascending = False, inplace=True)
    return rj, team_episodes
def saveEpisode(epid, rj):
    # request
    re = requests.post(get_url, json = {"EpisodeId": int(epid)})
    
    # save replay
    with open('{}.json'.format(epid), 'w') as f:
        f.write(re.json()['result']['replay'])

    # save episode info
    with open('{}_info.json'.format(epid), 'w') as f:
        json.dump([r for r in rj['result']['episodes'] if r['id']==epid][0], f)<jupyter_output><empty_output><jupyter_text>Scraper<jupyter_code>all_files = []
for root, dirs, files in os.walk('../input/', topdown=False):
    all_files.extend(files)
seen_episodes = [int(f.split('.')[0]) for f in all_files 
                      if '.' in f and f.split('.')[0].isdigit() and f.split('.')[1] == 'json']
print('{} games in existing library'.format(len(seen_episodes)))
pulled_teams = {}
pulled_episodes = []
start_time = datetime.datetime.now()
r = 0;

while len(pulled_episodes) < EPISODES:
    # pull team
    top_teams = [i for i in teams_df.id if i not in pulled_teams]
    if len(top_teams) > 0:
        team_id = top_teams[0]
    else:
        break;
        
    # get team data
    team_json, team_df = getTeamEpisodes(team_id); r+=1;
    print('{} games for {}'.format(len(team_df), teams_df.loc[teams_df.id == team_id].iloc[0].teamName))
    plt.hist(team_df.avg_score, bins = 150);
    
    team_df = team_df[team_df.avg_score > MIN_RATING]
    print('   {} with top competition'.format(len(team_df)))
    
    team_df = team_df[~team_df.id.isin(pulled_episodes + seen_episodes)]        
    print('      {} remain to be downloaded\n'.format(len(team_df)))
    
    # pull games
    target_team_games = int(np.ceil(EPISODES / NUM_TEAMS))
    if target_team_games + len(pulled_episodes) > EPISODES:
        target_team_games = EPISODES - len(pulled_episodes)
     
    pulled_teams[team_id] = 0
    
    i = 0
    while i < len(team_df) and pulled_teams[team_id] < target_team_games:
        epid = team_df.id.iloc[i]
        if not (epid in pulled_episodes or epid in seen_episodes):
            saveEpisode(epid, team_json); r+=1;
            pulled_episodes.append(epid)
            pulled_teams[team_id] += 1
            try:
                size = os.path.getsize('{}.json'.format(epid)) / 1e6
                print('Saved Episode #{} @ {:.1f}MB'.format(epid, size))
            except:
                print('  file {}.json did not seem to save'.format(epid))    
            if r > (datetime.datetime.now() - start_time).seconds:
                time.sleep( r - (datetime.datetime.now() - start_time).seconds)
                

        i+=1;
    print(); print()
print('\n   Post and share your datasets publicly on this thread:')
print('        https://www.kaggle.com/c/halite/discussion/164932\n')<jupyter_output>Post and share your datasets publicly on this thread:
        https://www.kaggle.com/c/halite/discussion/164932<jupyter_text>Check JSON<jupyter_code>file = './{}.json'.format(epid)
with open(file) as f:
    data = json.load(f)
data['rewards']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14006514657980457, 'fraction_numerical': 0.011879670434949224, 'mean_word_length': 3.335548172757475, 'pattern_counts': {'":': 3, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '293289', 'license': 'MIT', 'n_tokens_mistral': 1955, 'n_tokens_neox': 1878, 'n_words': 393, 'path': 'halite/notebooks/2-iwong-fetch-data-scraper.ipynb', 'repo_name': 'iainwo/kaggle', 'size': '5219'}