<jupyter_start><jupyter_text>Hidden cells with credentials for accessing parquets from Cloud Object Storage<jupyter_code># The code was removed by Watson Studio for sharing.
#cos = ibmos2spark.CloudObjectStorage(sc, credentials, configuration_name, 'bluemix_cos')

#movies_df = spark.read.parquet(cos.url('moviesdataframe.parquet', <BUCKET-NAME>))
movies_df.createOrReplaceTempView("movies")
result = spark.sql("""
            Select year as Year,COUNT(year) as Number_of_Movies
            FROM movies 
            GROUP BY year
            ORDER BY year ASC
                   """).repartition(1)

result.show()

## To save it locally
from IPython.display import HTML 
import base64

# function that takes in a dataframe and creates a text link to  
# download it (will only work for files < 2MB or so)
def create_download_link(df, title = "Download CSV file", filename = "result.csv"):  
    csv = df.to_csv(sep=',',header=True, index=False)
    b64 = base64.b64encode(csv.encode())
    payload = b64.decode()
    html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
    html = html.format(payload=payload,title=title,filename=filename)
    return HTML(html)


# sparkDF to Pandas
result_df = result.toPandas()
create_download_link(result_df,filename='movies_each_year.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11447562776957164, 'fraction_numerical': 0.011078286558345642, 'mean_word_length': 4.398406374501992, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '284832', 'license': 'Apache-2.0', 'n_tokens_mistral': 442, 'n_tokens_neox': 422, 'n_words': 110, 'path': 'SparkSQL/movies_released_each_year_sql.ipynb', 'repo_name': 'himasai97/MovieLens-Data-Analytics-in-Spark-and-Python', 'size': '1354'}