<jupyter_start><jupyter_text>On this website you can validate the structure of a JSON file: http://jsonlint.com/<jupyter_code>from pyspark.sql import SparkSession
spSession = SparkSession.builder.master('local').appName('appSparkSql').getOrCreate()
dfEmployees = spSession.read.json('aux/datasets/employees.json')
dfEmployees.show()
dfEmployees.printSchema()
dfEmployees.select('name').show()
dfEmployees.filter(dfEmployees['age'] > 40).orderBy(dfEmployees['age']).sort(dfEmployees['age'].desc()).show()
dfEmployees.groupBy('gender').count().show()
dfEmployees.groupBy('id_dep').agg({'salary': 'avg', 'age': 'max'}).show()
dfEmployees.registerTempTable('tt_employees')
spSession.sql(" \
    SELECT \
        id_dep, \
        MAX(age) AS MAX_AGE, \
        AVG(salary) AS SALARY_AVG \
    FROM tt_employees \
    GROUP BY id_dep").show()<jupyter_output>+------+-------+----------+
|id_dep|MAX_AGE|SALARY_AVG|
+------+-------+----------+
|     1|     42|    8100.0|
|     2|     50|    7500.0|
+------+-------+----------+
{'fraction_non_alphanumeric': 0.2323529411764706, 'fraction_numerical': 0.01764705882352941, 'mean_word_length': 3.4008620689655173, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '181762', 'license': 'MIT', 'n_tokens_mistral': 386, 'n_tokens_neox': 357, 'n_words': 52, 'path': 'modules/06-apache-spark-sql/03-json-files.ipynb', 'repo_name': 'cfascina/rtaps', 'size': '1020'}