<jupyter_start><jupyter_text><NAME>'s *Winning a Kaggle competition with Apache Spark and SparkML Machine Learning Pipelines*  Initialize Credentials and Path To Object Store<jupyter_code>import ibmos2spark

# @hidden_cell
credentials = {
    'auth_url': 'https://identity.open.softlayer.com',
    'project_id': '6aaf54352357483486ee2d4981f8ef15',
    'region': 'dallas',
    'user_id': 'b160340071b3407ca50c6b9a46b0bb25',
    'username': 'member_<PASSWORD>',
    'password': '<PASSWORD>['
}

configuration_name = 'os_d3bd5b94a9334de59a55a7fed2bedeaa_configs'
bmos = ibmos2spark.bluemix(sc, credentials, configuration_name)

from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()
# Please read the documentation of PySpark to learn more about the possibilities to load data files.
# PySpark documentation: https://spark.apache.org/docs/2.0.1/api/python/pyspark.sql.html#pyspark.sql.SparkSession
# The SparkSession object is already initalized for you.
# The following variable contains the path to your file on your Object Storage.
path_1 = bmos.url('dwlive', '4bertholdxor.py')<jupyter_output><empty_output><jupyter_text>Setup Cookies by capturing from your browser so we can download/wget files directly from kaggle<jupyter_code>#please create your own cookiescookies_kaggle = """
www.kaggle.com	FALSE	/	TRUE	1535524510	ai_user	6d0UU|2017-08-29T06:30.942Z
.kaggle.com	TRUE	/	FALSE	1506326405	__utmt	1
.kaggle.com	TRUE	/	FALSE	2137425820	intercom-id-koj6gxx6	d136e488-e012b2-917c-18e96cd11b53
.kaggle.com	TRUE	/	FALSE	1569397856	__utma	158690720.1103090056.1503510.1506325806.1506325806.1
.kaggle.com	TRUE	/	FALSE	1506327656	__utmb	158690720.3.9.1506356073
.kaggle.com	TRUE	/	FALSE	1522093856	__utmz	158690720.1506325.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)
.kaggle.com	TRUE	/	TRUE	1569397858	.ASPXAUTH	C03B6AF3A11A5A7B1643B1A3045BEA90749D3BDD9E7975FE2B5D993E3807E8A7EBA8014ADDEB41B8097DFE9573858D6F38B191B03049421AC5DCF41136A5520326698DF20838BBA9A357B65934C4FDD63D73D
.kaggle.com	TRUE	/	FALSE	1569397860	_ga	GA1.2.110309005503988510
.kaggle.com	TRUE	/	FALSE	1506412260	_gid	GA1.2.171929611506325781
www.kaggle.com	FALSE	/	TRUE	1506327661	ai_session	1TUXc|15325787951|1506325861052.695
.kaggle.com	TRUE	/	FALSE	1506930662	intercom-session-koj6gxx6	WG9FRmZRSEVUQjhPdHpTR0xIay9peVBYbzhvazlKNGJ4QzVzMztTnJiVVNiZzArNVRXZHVrcGJ3T0dXYy0tZ2x0S3BRUm53RW4vc3VVMGQ5WlFDUT09--c72537f5db077850756ed76403e3c4e6cf4bb981
"""
with open("cookies_kaggle.txt", "w") as text_file:
    text_file.write(cookies_kaggle)
! cat cookies_kaggle.txt<jupyter_output><empty_output><jupyter_text>Process train_numeric.csv.zip: get, unzip, read using DataFrames and save as parquet file for use later<jupyter_code>! wget --load-cookies cookies.txt https://www.kaggle.com/c/bosch-production-line-performance/download/train_numeric.csv.zip
!unzip train_numeric.csv.zip<jupyter_output><empty_output><jupyter_text>!ls -lahr train_numeric.csv<jupyter_code>!ls -lahr train_numeric.csv
df_numeric = spark.read.option("header",True).option("inferSchema", True).csv("train_numeric.csv")
df_numeric = df_numeric.repartition(1)
df_numeric.write.parquet(bmos.url('dwlive', 'train_numeric.parquet'))<jupyter_output><empty_output><jupyter_text>Process train_categorical.csv.zip: get, unzip, read using DataFrames and save as parquet file for use later<jupyter_code>! wget --load-cookies cookies.txt https://www.kaggle.com/c/bosch-production-line-performance/download/train_categorical.csv.zip
!unzip train_categorical.csv.zip
df_categorical = spark.read.option("header",True).option("inferSchema", True).csv("train_categorical.csv")
df_categorical = df_categorical.repartition(1)
df_categorical.write.parquet(bmos.url('BoschKaggleCompetition', 'train_categorical.parquet'))<jupyter_output><empty_output><jupyter_text>Test loading parquet File**Note: Replace URL path with your own**<jupyter_code># Please read the documentation of PySpark to learn more about the possibilities to load data files.
# PySpark documentation: https://spark.apache.org/docs/2.0.1/api/python/pyspark.sql.html#pyspark.sql.SparkSession
# The SparkSession object is already initalized for you.
# The following variable contains the path to your file on your Object Storage.
path_2 = bmos.url('dwlive', 'part-00000-abf604d6-bc0a-4ea3-8ffa-7838c5912fa2.snappy.parquet')
df_categorical = spark.read.parquet(path_2)
df_categorical.show()
df_categorical.printSchema()
!df -H<jupyter_output>Filesystem      Size  Used Avail Use% Mounted on
/dev/sda3       998G  658G  341G  66% /
devtmpfs        203G     0  203G   0% /dev
tmpfs           203G  1.9G  201G   1% /dev/shm
tmpfs           203G  4.4G  199G   3% /run
tmpfs           203G     0  203G   0% /sys/fs/cgroup
/dev/sdf        4.0T   93M  3.8T   1% /disk5
/dev/sdg        4.0T   93M  3.8T   1% /disk6
/dev/sdb        4.0T   11G  3.8T   1% /disk1
/dev/sdi        4.0T   93M  3.8T   1% /disk8
/dev/sdc        4.0T   93M  3.8T   1% /disk2
/dev/sdj        4.0T   93M  3.8T   1% /disk9
/dev/sdd        4.0T   93M  3.8T   1% /disk3
/dev/sde        4.0T   93M  3.8T   1% /disk4
/dev/sdh        4.0T   93M  3.8T   1% /disk7
/dev/sda1       265M  142M  124M  54% /boot
tmpfs            41G     0   41G   0% /run/user/0
/dev/fs01       271T  233T   38T  87% /gpfs/global_fs01
{'fraction_non_alphanumeric': 0.1008498583569405, 'fraction_numerical': 0.13257790368271954, 'mean_word_length': 4.448559670781893, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 5, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '67244', 'license': 'Apache-2.0', 'n_tokens_mistral': 2556, 'n_tokens_neox': 2208, 'n_words': 405, 'path': 'projects/bosch/ETLPython.ipynb', 'repo_name': 'kant/uhack', 'size': '5338'}