<jupyter_start><jupyter_text>--- Automatically generate new Python files from Sportradar API info page---<jupyter_code>import time
import json
import re
from selenium import webdriver<jupyter_output><empty_output><jupyter_text>Start the Selenium browser (Chrome)<jupyter_code># Start a new instance of Chrome
url = "https://developer.sportradar.com/io-docs"
browser = webdriver.Chrome()
browser.get(url)<jupyter_output><empty_output><jupyter_text>Get the method names and descriptions for a given API<jupyter_code>def getEndpointInfo():
    """Return a list with info on each endpoint listed on the Sportradar API info page.
    The method will only return info on an API that is currently displayed on page.
    """
    list_of_endpoints = browser.find_elements_by_class_name('endpointList')
    for n,item in enumerate(list_of_endpoints):
        eps = item.find_element_by_class_name('endpoint')
        title = eps.find_element_by_class_name('title')

        name = title.find_element_by_class_name('name')
        if name.text != "":
            print(name.text + '\n'+str(n))

            # Expand all endpoint descriptions.
            actions = title.find_element_by_class_name('actions')
            expand = actions.find_element_by_class_name('expand-methods')
            expand.click()
            time.sleep(0.5)

            # Scrape the GET method names, URIs, and descriptions
            endpoints = []
            method_list = eps.find_element_by_class_name("methods")
            methods = method_list.find_elements_by_class_name('method')
            for method in methods:
                m = method.find_element_by_class_name('title')

                # Get the name of and URI for the current endpoint
                name = m.find_element_by_class_name('name').text
                uri = m.find_element_by_class_name('uri').text

                # Get the endpoint's description 
                h = method.find_element_by_class_name('hidden')
                d = method.find_element_by_class_name('description')    
                p = d.find_element_by_tag_name('p')
                description = p.text    
                                
                # Get default values for the parameters for each method (for tests)
                table_items = h.find_elements_by_class_name('type-pathReplace')
                param_names, param_values = [], []
                for t in table_items[:-1]:
                    param_name = t.find_element_by_class_name('name')
                    p_name = param_name.get_attribute('textContent').strip().split(':',1)[-1]
                    param_names.append(p_name)
                    if p_name in ['year', 'month', 'day']:
                        options = t.find_elements_by_tag_name('option')
                        for o in options:
                            if o.get_property('selected'):
                                param_values.append(o.get_property('value'))                                
                    else:                        
                        try:
                            options = t.find_elements_by_tag_name('option')
                            param_values.append(t.find_elements_by_tag_name('option')[-1].get_property('value'))                     
                        except:
                            param_values.append(t.find_elements_by_tag_name('input')[-1].get_property('value'))

                default_params = dict(zip(param_names, param_values))
                
                endpoints.append({'name':name, 'uri': uri,
                      'description': description, 'defaults': default_params})
                print('-'*20)
                print(default_params)                
                print('{n}: {u}\n{d}'.format(n=name, u=uri, d=description))        
                
    return endpoints<jupyter_output><empty_output><jupyter_text>Get information on all possible APIs provided by Sportradar<jupyter_code>api_names_to_scrape = ['Beach Volleyball Trial', 'Darts Trial', 'eSports Dota 2 Trial', 'Global Basketball Trial', 'Global Ice Hockey Trial', 'Golf Trial', 'eSports LoL Trial', 'MLB v6.5 Trial', 'NASCAR Official Trial', 'NBA Official Trial', 'NFL Official Trial v2', 'NHL Official Trial', 'Rugby v2 API', 'Soccer EU Trial v3', 'Tennis v2 Trial', 'WNBA Trial']
services = browser.find_element_by_class_name("services")

# Names of the APIs
api_selector = services.find_element_by_tag_name('h2')
selector = api_selector.find_element_by_tag_name('select')
options = selector.find_elements_by_tag_name('option')[1:]

# Descriptions of the APIs
api_elems = services.find_elements_by_class_name('apiDescriptionList')

# Extract the names and descriptions of the desired APIs
apis = []
for api, option in zip(api_elems, options):
    name = option.text    
    
    if name in api_names_to_scrape:
        description = api.get_attribute('textContent').strip()
        print("-----------\n{n}: {d}".format(n=name, d=description))
        option.click() # Select the API, changing the displayed methods
        time.sleep(1)
        
        # Extract the endpoint names and descriptions for the current API
        endpoint_info = getEndpointInfo()     
        apis.append({'name': name, 'description': description, 'endpoints': endpoint_info})

# Save the API information
filename = 'api_names_and_endpoints'
with open(filename + '.json', 'w') as outfile:
    json.dump(apis, outfile)<jupyter_output>-----------
Beach Volleyball Trial: Sportradar's Beach Volleyball API
Beach Volleyball
15
--------------------
{'competitor_id': 'sr:competitor:151994'}
Competitor Profile: volleyball-t1/beach/en/competitors/:competitor_id/profile:format
Provides information for a given competitor
--------------------
{'competitor_id': 'sr:competitor:151994'}
Competitor Results: volleyball-t1/beach/en/competitors/:competitor_id/results:format
Provides past match results for a given competitor
--------------------
{'competitor_id': 'sr:competitor:151994'}
Competitor Schedule: volleyball-t1/beach/en/competitors/:competitor_id/schedule:format
Provides the schedule for a given competitor
--------------------
{'year': '2018', 'month': '06', 'day': '01'}
Daily Results: volleyball-t1/beach/en/schedules/:year-:month-:day/results:format
Provides the match scoring for all matches played on a given day
--------------------
{'year': '2018', 'month': '06', 'day': '01'}
Daily Schedule: volleyball-t1/beach/en/schedul[...]<jupyter_text>Load the saved API data<jupyter_code>apis = json.load(open('api_names_and_endpoints.json', 'rb'))<jupyter_output><empty_output><jupyter_text>Generate new Python classes and files from API info<jupyter_code>apis[0].keys()
[a['name'] for a in apis]
endpoints = apis[10]['endpoints']
endpoints[2]
uri = endpoints[2]['uri']
uri
max_width = 79
def formatArgListAndPathFromURI(uri, format_for_tests=False):
    uri = uri.replace('//','/') # Deal with a typo on Sportradar
    regex = re.compile('(?!:format)(:[\w_]+)', re.VERBOSE)    
    
    # Identify the arguments to the URI, excluding :format
    parameters = [s.strip(':') for s in regex.findall(uri) if s != '']
    if format_for_tests:
        arg_list = ''
        for p in parameters:
            arg_list += "self.{p}, ".format(p=p)
        if parameters:
            arg_list = arg_list.strip(', ')
    else:
        arg_list = ", ".join(parameters)
    
    # Create the path string, allowing user to substitute in arguments
    path = (regex.sub('{\g<1>}', uri).split(':format')[0] + '"').replace('{:','{')
    print(path)
    
    # Make sure dates are formatted properly
    if 'year' in parameters:
        path = path.replace('{year}', '{year:4d}')
    if 'month' in parameters:
        path = path.replace('{month}', '{month:02d}')
    if 'day' in parameters:
        path = path.replace('{day}', '{day:02d}')        
    
    # Append .format() to the end of the path string
    format_suffix = ""
    for p in parameters:
        format_suffix += "{arg}={val}, ".format(arg=p, val=p)
    format_suffix = ".format({})".format(format_suffix.strip(', '))    
    path += format_suffix
#     path = path.split('/',3)[-1]
    
    # Comply to max width of lines
    if len(path) > max_width-8:
        regex = re.compile(r'\.format\(', re.VERBOSE)
        path = regex.sub(r'.format(\n\t\t\t', path)        
    path = '"' + path
    
    return arg_list, path

def formatPathFromURI(uri):
    parameters = [s.strip(':') for s in re.findall(r'(:[\w_]+)*', uri) if s != ''][:-1]

def formatDocString(doc, mw=79):
    if len(doc) > mw-7:
        idx_cut = (mw-len(doc[mw::-1].split(' ')[0])) # kludge        
        doc = doc[:idx_cut] + '\n\t\t\t' + formatDocString(doc[idx_cut:].strip(' '), mw)
    return doc

def paramValsAreComplete(endpoint):
    for val in endpoint['defaults'].values():
        if val == '':
            return False
    return True

def formatDefaultParamVals(endpoints):
    assert type(endpoints)==list, 'Must provide a list of endpoints'
    varlist = ''
    used_vars = []
    
    for ep in endpoints:
        defaults = ep['defaults']
        for var,val in defaults.items():
            if var not in used_vars and val != '':
                if var in ['year', 'month', 'day']:                    
                    varlist += '\t\tcls.{var} = {val}\n'.format(var=var, val=int(val))
                else:
                    varlist += '\t\tcls.{var} = "{val}"\n'.format(var=var, val=val)
                used_vars.append(var)
    return varlist

def formatClassName(name):
    name = name.replace(' Trial','').replace('Official','')
    name = name.replace(' ','').replace('.','_').replace('(','')
    name = name.replace(')','').replace('-','_').replace(',','_')
    return name
    
def writeToPythonFile(api_info):
    """Write methods based on the endpoint attributes
    :param api_info: (dict) Contains API name, description and list of endpoints
    """

    # --------------------------------------------------------
    #    Write the methods
    # --------------------------------------------------------
                
    # Class name    
    class_name = formatClassName(api_info['name'])
    regex = re.compile(r'v[\d_]+', re.IGNORECASE)
    class_name = regex.sub('', class_name)
        
    # Add comment heading and import statement
    txt = "# Sportradar APIs\n# Copyright 2018 <NAME>\n# See LICENSE for details.\n\n"
    txt += "from sportradar.api import API\n\n\n"
           
    # Add class name to file   
    txt += "class {}(API):\n\n".format(class_name)
    
    # Add __init__ function to class
    txt += "\tdef __init__(self, api_key, format_='json', timeout=5, sleep_time=1.5):\n"
    txt += "\t\tsuper().__init__(api_key, format_, timeout, sleep_time)\n\n"
    txt = txt.replace('\t', 4*' ')
            
    # Write the method, including arguments, doc string, and URI path    
    endpoints = api_info['endpoints']    
    for n,ep in enumerate(endpoints):
        mn = 'get_' + formatClassName(ep['name']).lower()
        doc = formatDocString(ep['description'], max_width)
        doc = doc + '"""' if len(doc) < max_width-8 else doc + '\n\t\t"""'
        args, path = formatArgListAndPathFromURI(ep['uri'])                
        arglist = 'self, {a}'.format(a=args) if args != '' else 'self'                
        
        # Assemble the method string
        txt += '\tdef {method_name}({args}):\n'.format(method_name=mn, args=arglist)
        txt += '\t\t"""{doc}\n\t\tpath = {path}\n'.format(doc=doc, path=path)
        txt += '\t\treturn self._make_request(path)\n\n'
        txt = txt.replace('\t', 4*' ')
                        
    print('-'*20)
    print(api_info['name'])
#     print(path)
#     print(txt)
    
    # Save to Python file
    filename = class_name + '.py'
    with open(filename, 'w+') as pyfile:
        pyfile.write(txt)
        
        
    # --------------------------------------------------------
    #    Write the tests
    # --------------------------------------------------------
    
    # Write the test front matter
    txt = "import os\nimport unittest\nfrom sportradar import {}\n\n".format(class_name)
    txt += '# Import API keys from environment variables\n'
    txt += 'api_key_name = "SPORTRADAR_API_KEY_{}"\n'.format(class_name.upper())
    txt += 'api_key = os.environ.get(api_key_name, None)\n'
    txt += 'assert api_key is not None, "Must declare environment variable: {key_name}".format(\n'
    txt += '\tkey_name=api_key_name)\n'
    txt += 'api = {}.{}(api_key, format_="json", timeout=5, sleep_time=1.5)\n\n\n'.format(
        class_name, class_name)
    txt += 'class TestAPI(unittest.TestCase):\n\n'
    txt += '    @classmethod\n    def setUpClass(cls):\n'
    txt += '\t\tprint("\\n---------------------\\nSetting up {} tests...\\n".format("{}"))\n'.format("{}", class_name)
    txt += '\t\tcls.auth = api_key\n\t\tcls.api = api\n'
    txt += '{defaults}\n'.format(defaults=formatDefaultParamVals(endpoints))
       
    # Write the test methods
    # Write the method, including arguments, doc string, and URI path    
    for n,ep in enumerate(endpoints):        
        if paramValsAreComplete(ep):        
            mn = 'test_get_' + formatClassName(ep['name']).lower()
            doc = '"""Test the {} GET query"""'.format(ep['name'].lower())    
            args, path = formatArgListAndPathFromURI(ep['uri'], format_for_tests=True)
            arglist = '{a}'.format(a=args) if args != '' else ''
            print(arglist)

            # Assemble the method string    
            txt += '\tdef {method_name}(self):\n\t\t{doc}\n\t\tmsg = "Response status is not 200"\n'.format(
                    method_name=mn, doc=doc)
            txt += '\t\tresponse = self.api.{}({})\n'.format(
                mn.split('test_')[1], arglist.split('self, ', 1)[-1])
            
            if n == len(endpoints)-1:            
                txt += '\t\tself.assertEqual(response.status_code, 200, msg)\n'
            else:
                txt += '\t\tself.assertEqual(response.status_code, 200, msg)\n\n'
            txt = txt.replace('\t', 4*' ')
        
    # Save to Python file
    filename = 'test_' + class_name + '.py'
    with open(filename, 'w+') as pyfile:
        pyfile.write(txt)        

# api_names_to_scrape = ['Beach Volleyball Trial', 'Darts Trial', 'Golf Trial', 'MLB v6.5 Trial', 'NASCAR Official Trial', 'NBA Official Trial', 'NFL Official Trial v2', 'NHL Official Trial', 'Tennis v2 Trial', 'WNBA Trial']
api_names_to_scrape = ['eSports Dota 2 Trial']
for api in apis:
    if api['name'] in api_names_to_scrape:
        writeToPythonFile(api)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16092110641776808, 'fraction_numerical': 0.007971113230685379, 'mean_word_length': 2.3513698630136988, 'pattern_counts': {'":': 1, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '500011', 'license': 'MIT', 'n_tokens_mistral': 4616, 'n_tokens_neox': 4444, 'n_words': 1044, 'path': 'assets/code/scrapeSportradarEndpoints.ipynb', 'repo_name': 'johnwmillr/johnwmillr.github.io', 'size': '14686'}