<jupyter_start><jupyter_text>Using Spark from Livy end point
The samples demostrates using spark service via the livy end point. The sample uses the python request library to access the livy interface. 
Livy API details are available at https://livy.incubator.apache.org/docs/latest/rest-api.html
** Note : The image here may not be visible dues to markdown bug. Please change path here to full path to view the image.


  Using python requests library to access restful APIs<jupyter_code>import json, pprint, requests, textwrap
from requests.auth import HTTPBasicAuth
print("Get username and email ")
print("---------------------- ")

#Basic test for request
import requests

r = requests.get('https://jsonplaceholder.typicode.com/users')
r.encoding = 'utf-8'
ret_json = r.json()


for user in ret_json:
    print(user['username'] , user['email'])

#print(r.json())<jupyter_output>Get username and email 
----------------------<jupyter_text>Livy APIs to fetch running sessions and sesssion state<jupyter_code>import json, pprint, requests, textwrap
from requests.auth import HTTPBasicAuth

#Diabling exception to avoid the https verificatin warning
import requests.packages.urllib3 as urllib3
import urllib3.exceptions as urllib3_exceptions       
urllib3.disable_warnings(urllib3_exceptions.InsecureRequestWarning)


#Change host per your confirgration                               
host = "https://<ip>:<port>/gateway/default/livy/v1"

#Livy interface as per https://livy.incubator.apache.org/docs/latest/rest-api.html#session

# Construct Request - Get a list of current spark sessions.
sessions_url = host + "/sessions"

# Common headers for all requests.
# Auth header
auth = HTTPBasicAuth("***", "***")
# Content Type
headers = {'Content-Type': 'application/json'}

data = {
        'from': 0, 
        'size': 10
       }
r = requests.get(sessions_url, data=json.dumps(data), headers=headers, auth=auth, verify=False)

response_body = r.json()

print("Sessions fetched starting ", response_body['from'])
print("Number of session fetched", response_body['total'])
session_list = response_body['sessions']

for session in session_list:
    print("The session {0} has a state {1} ".format(session['id'],session['state']))<jupyter_output>Sessions fetched starting  0
Number of session fetched 1
The session 10 has a state idle<jupyter_text>Create a new Spark session and query state<jupyter_code>#Create a new Spark session
data = {
    'kind':'pyspark'
}

r = requests.post(sessions_url, data=json.dumps(data), headers=headers, auth=auth, verify=False)
response_body = r.json()

session_id = response_body['id']
session_state = response_body['state']
print("Spark session {0} created. Current state is {1}".format(session_id,session_state))
created_session_url = r.headers['location']




#Query information about the session we just created
this_session_url = host + created_session_url
print("this_session_url", this_session_url)

r = requests.get(this_session_url, headers=headers,auth=auth, verify=False)
pprint.pprint(r.json())<jupyter_output>this_session_url https://172.16.58.3:30443/gateway/default/livy/v1/sessions/13<jupyter_text>Execute code interactively Spark session and check results<jupyter_code>#Execute code interactively in this session using session/<ID>/statements interface

statements_url = this_session_url + "/statements"
pprint.pprint(statements_url)

data = {
  'code': "11+11"
}

r = requests.post(statements_url, data=json.dumps(data), headers=headers, auth=auth, verify=False)
print("Respone status code" ,r.status_code )
print("Response received is ",r.json())
print("Poll URI is ",r.headers['location'])
#response_body = r.json()




# Wait a while before executing this cell. Spark session start up takes time to run your code.
specific_statement = host + r.headers['location']
print("monitoring url is ", specific_statement)

r = requests.get(specific_statement, headers=headers, auth=auth, verify=False)
print("Response status is ",r.status_code)
print("Response received is ", pprint.pprint(r.json()))<jupyter_output>monitoring url is  https://172.16.58.3:30443/gateway/default/livy/v1/sessions/13/statements/0<jupyter_text>Execute code in batch<jupyter_code>#Execute code in batch
#The following uses pi.py from spark source. Please get that file and transer to HDFS /jar folder.
batch_url = host + "/batches"
print("batch_url", batch_url)

data = {
    'file' : '/jar/pi.py'    
}

r = requests.post(batch_url, data=json.dumps(data), headers=headers, auth=auth, verify=False)
returned_batch_url = r.headers['location']
print("Respone status code" , r.status_code)
print("Poll URI is ",returned_batch_url )
print("Response is ", pprint.pprint(r.json()))
#Check results of executed code
specific_batch = host + returned_batch_url
print("specific batch request ",specific_batch)

r = requests.get(specific_batch,headers=headers, auth=auth, verify = False)
print("Response status is ",r.status_code)
print("Response received is ", pprint.pprint(r.json()))<jupyter_output>specific batch request  https://172.16.58.3:30443/gateway/default/livy/v1/batches/1
{'fraction_non_alphanumeric': 0.12115975422427035, 'fraction_numerical': 0.013632872503840246, 'mean_word_length': 4.625269978401728, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '54199', 'license': 'MIT', 'n_tokens_mistral': 1717, 'n_tokens_neox': 1643, 'n_words': 439, 'path': 'samples/features/sql-big-data-cluster/spark/restful-api-access/accessing_spark_via_livy.ipynb', 'repo_name': 'KarlMaynardDK/sql-server-samples', 'size': '5208'}