<jupyter_start><jupyter_text>Building a csv of all your PBs-- a short story by baldnate  Get user id<jupyter_code>import json
import pandas as pd
import requests
import math

users = {}
def getUserId(username):
  if username not in users:
    url = "https://www.speedrun.com/api/v1/users?name=" + username
    data = requests.get(url).json()['data']
    if len(data) == 1:
      users[username] = data[0]['id']
    else:
      raise Exception('Searched for ' + username + ', got back ' + str(len(data)) + ' entries (expected 1)') 
  return users[username]

userid = getUserId('baldnate')<jupyter_output><empty_output><jupyter_text>Get PBs<jupyter_code>def getPBs(userid, all = False):
  url = "https://www.speedrun.com/api/v1/users/" + userid + "/personal-bests?embed=game,category,region,platform,players"
  data = requests.get(url).json()['data']
  pbdf = pd.DataFrame(data)
  pbdf = pbdf.join(pbdf['run'].apply(pd.Series), rsuffix='run')
  pbdf.drop(axis=1, columns=['run'], inplace=True)
  if all:
    allurl = "https://www.speedrun.com/api/v1/runs?user=" + userid + "&embed=game,category,region,platform,players"
    data = requests.get(allurl).json()['data']
    alldf = pd.DataFrame(data)
    alldf['place'] = math.nan
    pbdf = pbdf.append(alldf)
    return pbdf
  return pbdf

rawdf = getPBs(userid, all=True)
runsdf = pd.DataFrame()
rawdf.tail()<jupyter_output><empty_output><jupyter_text>Co-Op - aka: write player(s) to a column  "Simple" Columns<jupyter_code>runsdf['place'] = rawdf['place']
runsdf['gamename'] = rawdf.apply(lambda x: x.game['data']['names']['international'], axis=1)
runsdf['categoryname'] = rawdf.apply(lambda x: x.category['data']['name'], axis=1)
runsdf['time'] = rawdf.apply(lambda x: x.times['primary_t'], axis=1)
runsdf['date'] = rawdf.apply(lambda x: x.date, axis=1)
runsdf['video'] = rawdf.apply(lambda x: x.videos['links'][0]['uri'], axis=1)
runsdf['comment'] = rawdf.apply(lambda x: str(x.comment).replace('\n', ' ').replace('\r', ' ') , axis=1)<jupyter_output><empty_output><jupyter_text>Columns that need optional handling<jupyter_code>def getRegion(x):
  if len(x.region['data']) == 0:
    return "None"
  else:
    return x.region['data']['name']

runsdf['regionname'] = rawdf.apply(lambda x: getRegion(x), axis=1)

def getPlatform(x):
  if len(x.platform['data']) == 0:
    return "None"
  else:
    return x.platform['data']['name']

runsdf['platformname'] = rawdf.apply(lambda x: getPlatform(x), axis=1)<jupyter_output><empty_output><jupyter_text>Sub-CategoriesMemoized for speed and kindness.<jupyter_code>varMemo = {}
def getVariable(variableid):
  if variableid not in varMemo:
    url = "https://www.speedrun.com/api/v1/variables/" + variableid
    response = requests.get(url)
    varMemo[variableid] = response.json()['data']
  return varMemo[variableid]

def getValue(variableid, valueid):
  var = getVariable(variableid)
  return var['values']['values'][valueid]['label']

def getSubCategories(x):
  if x['values'] == {}:
    return "None"
  else:
    vals = []
    for varid, valid in x['values'].items():
      if getVariable(varid)['is-subcategory']:
        vals.append(getValue(varid, valid))  
    return " -- ".join(vals)

runsdf['subcategories'] = rawdf.apply(lambda x: getSubCategories(x), axis=1)<jupyter_output><empty_output><jupyter_text>Dump to a csv<jupyter_code>def getPlayers(x):
  players = []
  for p in x.players['data']:
    players.append(p['names']['international']) 
  return ", ".join(players)

runsdf['players'] = rawdf.apply(lambda x: getPlayers(x), axis=1)
import csv
runsdf.fillna(value="").to_csv('runs.csv', index=False, quoting=csv.QUOTE_NONNUMERIC)
print('csv exported')
csvdf = pd.read_csv('runs.csv')
csvdf.tail()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17921242968122153, 'fraction_numerical': 0.005625502276989017, 'mean_word_length': 5.361158432708688, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '667315', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1397, 'n_tokens_neox': 1379, 'n_words': 250, 'path': 'playground.ipynb', 'repo_name': 'baldnate/src-pbs-to-csv', 'size': '3733'}