<commit_before>import sys
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import json


INSTANCE_ID = sys.argv[1]
JOB_NAME = sys.argv[2]
BRANCH = sys.argv[3]
JOB_PARAMETERS = sys.argv[4]
ENV = sys.argv[5]
FILE_LOCATION = sys.argv[6]

# Run Get request with api_command param
# /jobs/list/ with api 2.0 returns all jobs, 2.1 does not
def getRequest(api_command, params={}):
    if api_command == "/jobs/list":
        url = "https://{}{}{}".format(INSTANCE_ID, "/api/2.0", api_command)
    else:
        url = "https://{}{}{}".format(INSTANCE_ID, API_VERSION, api_command)
    response = requests.get(
      url = url,
      json = params,
    )
    return response

# Get all job names and jobID"s and map to dict
def getJobIds(res):
    tempDict = {}
    for job in res.json()["jobs"]:
      tempDict[job["settings"]["name"]] = job["job_id"]
    return tempDict


def updateJsonFile(fileName):
    # Open the JSON file for reading
    jsonFile = open(fileName, "r") 
    data = json.load(jsonFile) 
    jsonFile.close()

    # Edit content
    # Set notebook params for job
    python_params = JOB_PARAMETERS.split("\n")
    env_vars = {
        "DATABASE_URL": "{{secrets/" + ENV + "/DATABASE_URL}}",
        "BRANCH": BRANCH,
        "ENV_CODE": ENV 
    }
    data["tasks"][0]["spark_python_task"]["python_file"] = "dbfs:/FileStore/" + BRANCH + "/manage.py"
    data["tasks"][0]["spark_python_task"]["parameters"] = python_params
    data["tasks"][0]["new_cluster"]["spark_env_vars"] = env_vars
    data["tasks"][0]["new_cluster"]["cluster_name"] = JOB_NAME

    ## Save our changes to JSON file
    jsonFile = open(fileName, "w+")
    jsonFile.write(json.dumps(data))
    jsonFile.close()


# Start script
jobs = getJobIds(getRequest("/jobs/list"))

if( JOB_NAME in jobs ):
    sys.stdout.write( (str(jobs[JOB_NAME])) )

    updateJsonFile(FILE_LOCATION)




<commit_msg>update cluster cant have job name<commit_after>import sys
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import json


INSTANCE_ID = sys.argv[1]
JOB_NAME = sys.argv[2]
BRANCH = sys.argv[3]
JOB_PARAMETERS = sys.argv[4]
ENV = sys.argv[5]
FILE_LOCATION = sys.argv[6]

# Run Get request with api_command param
# /jobs/list/ with api 2.0 returns all jobs, 2.1 does not
def getRequest(api_command, params={}):
    if api_command == "/jobs/list":
        url = "https://{}{}{}".format(INSTANCE_ID, "/api/2.0", api_command)
    else:
        url = "https://{}{}{}".format(INSTANCE_ID, API_VERSION, api_command)
    response = requests.get(
      url = url,
      json = params,
    )
    return response

# Get all job names and jobID"s and map to dict
def getJobIds(res):
    tempDict = {}
    for job in res.json()["jobs"]:
      tempDict[job["settings"]["name"]] = job["job_id"]
    return tempDict


def updateJsonFile(fileName):
    # Open the JSON file for reading
    jsonFile = open(fileName, "r") 
    data = json.load(jsonFile) 
    jsonFile.close()

    # Edit content
    # Set notebook params for job
    python_params = JOB_PARAMETERS.split("\n")
    env_vars = {
        "DATABASE_URL": "{{secrets/" + ENV + "/DATABASE_URL}}",
        "BRANCH": BRANCH,
        "ENV_CODE": ENV 
    }
    data["tasks"][0]["spark_python_task"]["python_file"] = "dbfs:/FileStore/" + BRANCH + "/manage.py"
    data["tasks"][0]["spark_python_task"]["parameters"] = python_params
    data["tasks"][0]["new_cluster"]["spark_env_vars"] = env_vars

    ## Save our changes to JSON file
    jsonFile = open(fileName, "w+")
    jsonFile.write(json.dumps(data))
    jsonFile.close()


# Start script
jobs = getJobIds(getRequest("/jobs/list"))

if( JOB_NAME in jobs ):
    sys.stdout.write( (str(jobs[JOB_NAME])) )

    updateJsonFile(FILE_LOCATION)





{'fraction_non_alphanumeric': 0.15281076801266824, 'fraction_numerical': 0.00870942201108472, 'mean_word_length': 3.5498199279711886, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2461034', 'n_tokens_mistral': 1349, 'n_tokens_neox': 1329, 'n_words': 295}