def task(sc, i):
    print(sc.parallelize(list(range(i * 10000))).count())

def run_multiple_jobs():
    conf = SparkConf().setMaster('local[*]').setAppName('appname')
conf.set('spark.scheduler.mode', 'FAIR')
sc = SparkContext(conf=conf)
for i in range(4):
    t = threading.Thread(target=task, args=(sc, i))
t.start()
print('spark task', i, 'has started')
run_multiple_jobs()
How to run multiple jobs in one sparkcontext from separate threads in pyspark?