<commit_before>import os
import pandas as pd 
import requests
import re

os.environ['http_proxy']=""

class SparkPyRest(object): 
    """Basic class for handling Spark REST API queries"""

    def __init__(self, host, port=4040): 
        self.host = host
        self.port = port
        self.base_url = base_url = 'http://{host}:{port}/api/v1'.format(host=host, port=port) 

    @property
    def app(self): 
        return get_app(self.base_url)


    @property
    def stages(self): 
        return get_stages(self.base_url)


    @property
    def executors(self):
        response = requests.get(self.base_url+'/applications/' + self.app + '/executors')
        executors = response.json()
        return executors


    def tasks(self, stageid):
        """Return the tasks for a given stageid"""
        return get_tasks(self.base_url, stageid)


    def executor_log_bytes(self, executor_id):
        """Find out how big the executor log file is"""
        executors = self.executors

        for executor in executors: 
            if executor['id'] == executor_id:
               break

        log_url = executors[executor_id]
        response = requests.get(executor['executorLogs']['stderr'].replace('logPage', 'log')+'&offset=0')
        total_bytes = int(re.findall('\d+\sof\s(\d+)\sof', response.text)[0])
        return total_bytes        


    def executor_log(self, executor_id, nbytes='all'):
        """Fetch the executors log; by default, the entire log is returned."""
        total_bytes = self.executor_log_bytes(executor_id)

        executors = self.executors

        for executor in executors: 
            if executor['id'] == executor_id:
               break

        if nbytes=='all':
            nbytes = total_bytes

        else: 
            if not isinstance(nbytes,int): 
                raise RuntimeError('nbytes must be an integer')

        query = executor['executorLogs']['stderr'].replace('logPage', 'log')
        query += '&offset=0'
        query += '&byteLength={}'.format(nbytes)
        response = requests.get(query)

        return response.text


# base public functions

def get_app(base_url):
    """Get the app ID from the REST server"""
    response = requests.get(base_url+'/applications')
    return response.json()[0]['id']


def get_stages(base_url):
    """Get stage IDs"""
    response = requests.get(base_url+'/applications/'+get_app(base_url)+'/stages')
    stages = response.json()
    return [(stage['stageId'], stage['name']) for stage in stages]
<commit_msg>fixed bug with executor_id type<commit_after>import os
import pandas as pd 
import requests
import re

os.environ['http_proxy']=""

class SparkPyRest(object): 
    """Basic class for handling Spark REST API queries"""

    def __init__(self, host, port=4040): 
        self.host = host
        self.port = port
        self.base_url = base_url = 'http://{host}:{port}/api/v1'.format(host=host, port=port) 

    @property
    def app(self): 
        return get_app(self.base_url)


    @property
    def stages(self): 
        return get_stages(self.base_url)


    @property
    def executors(self):
        response = requests.get(self.base_url+'/applications/' + self.app + '/executors')
        executors = response.json()
        return executors


    def tasks(self, stageid):
        """Return the tasks for a given stageid"""
        return get_tasks(self.base_url, stageid)


    def executor_log_bytes(self, executor_id):
        """Find out how big the executor log file is"""
        executors = self.executors

        for executor in executors: 
            if executor['id'] == executor_id:
               continue

        log_url = executors[executor_id]
        response = requests.get(executor['executorLogs']['stderr'].replace('logPage', 'log')+'&offset=0')
        total_bytes = int(re.findall('\d+\sof\s(\d+)\sof', response.text)[0])
        return total_bytes        


    def executor_log(self, executor_id, nbytes='all'):
        """Fetch the executors log; by default, the entire log is returned."""
        total_bytes = self.executor_log_bytes(executor_id)

        executors = self.executors

        for executor in executors: 
            if executor['id'] == str(executor_id):
               break

        if nbytes=='all':
            nbytes = total_bytes

        else: 
            if not isinstance(nbytes,int): 
                raise RuntimeError('nbytes must be an integer')

        query = executor['executorLogs']['stderr'].replace('logPage', 'log')
        query += '&offset=0'
        query += '&byteLength={}'.format(nbytes)
        response = requests.get(query)

        return response.text


# base public functions

def get_app(base_url):
    """Get the app ID from the REST server"""
    response = requests.get(base_url+'/applications')
    return response.json()[0]['id']


def get_stages(base_url):
    """Get stage IDs"""
    response = requests.get(base_url+'/applications/'+get_app(base_url)+'/stages')
    stages = response.json()
    return [(stage['stageId'], stage['name']) for stage in stages]

{'fraction_non_alphanumeric': 0.12853773584905662, 'fraction_numerical': 0.003537735849056604, 'mean_word_length': 2.6850108616944244, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1823772', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1501, 'n_words': 372}