<jupyter_start><jupyter_text>Starting the Spark Session<jupyter_code>spark = SparkSession.builder.appName('dateml').getOrCreate()<jupyter_output><empty_output><jupyter_text>Loading the Data<jupyter_code>df = spark.read.csv('../tutorial-data/ClaimDataExample1.csv', header=True, inferSchema=True)
df.printSchema()<jupyter_output>root
 |-- CLM_NO: integer (nullable = true)
 |-- CLM_NO_LN: integer (nullable = true)
 |-- RCV_DT: timestamp (nullable = true)
 |-- SERV_DT: timestamp (nullable = true)
 |-- RESOLVED_DT: timestamp (nullable = true)
 |-- ALLOW_AMT: double (nullable = true)
 |-- RESOLVED_AMT: double (nullable = true)<jupyter_text>Registering the Table<jupyter_code>df.registerTempTable('ClaimMock')
spark.catalog.listTables()<jupyter_output><empty_output><jupyter_text>Auto-selecting Date Fields<jupyter_code>date_columns = [i[0] for i in list(filter(lambda x: x[1]=='timestamp', df.dtypes))]
df.select(date_columns).show()
df.withColumn('duration', datediff(df['RCV_DT'], df['SERV_DT'])).show()<jupyter_output>+--------+---------+-------------------+-------------------+-------------------+------------------+------------------+--------+
|  CLM_NO|CLM_NO_LN|             RCV_DT|            SERV_DT|        RESOLVED_DT|         ALLOW_AMT|      RESOLVED_AMT|duration|
+--------+---------+-------------------+-------------------+-------------------+------------------+------------------+--------+
|10000000|        1|2017-02-04 00:00:00|2017-02-14 00:00:00|2017-03-16 00:00:00| 63.06704903424165| 8.829386864793833|     -10|
|10000000|        2|2017-07-08 00:00:00|2017-07-18 00:00:00|2017-08-17 00:00:00| 25.39506690821225|3.5553093671497153|     -10|
|10000000|        3|2017-09-15 00:00:00|2017-09-25 00:00:00|2017-10-25 00:00:00| 901.7757618741892| 126.2486066623865|     -10|
|10000000|        4|2018-03-03 00:00:00|2018-03-13 00:00:00|2018-04-12 00:00:00| 967.1990971331805| 135.4078735986453|     -10|
|10000000|        5|2018-08-04 00:00:00|2018-08-14 00:00:00|2018-09-13 00:00:00|325.94044087681294| 45.6[...]<jupyter_text>Feature Engineering on Date Fields<jupyter_code># create all posible combinations
i = 0
while i < len(date_columns):    
    target_col = date_columns[i]
    for col in date_columns:
        if target_col != col:
            df = df.withColumn(f'{target_col}_to_{col}_diff', datediff(target_col, col))
    i += 1
df.select(list(filter(lambda x: 'diff' in x, df.columns))).show()<jupyter_output>+----------------------+--------------------------+----------------------+---------------------------+--------------------------+---------------------------+
|RCV_DT_to_SERV_DT_diff|RCV_DT_to_RESOLVED_DT_diff|SERV_DT_to_RCV_DT_diff|SERV_DT_to_RESOLVED_DT_diff|RESOLVED_DT_to_RCV_DT_diff|RESOLVED_DT_to_SERV_DT_diff|
+----------------------+--------------------------+----------------------+---------------------------+--------------------------+---------------------------+
|                   -10|                       -40|                    10|                        -30|                        40|                         30|
|                   -10|                       -40|                    10|                        -30|                        40|                         30|
|                   -10|                       -40|                    10|                        -30|                        40|                         30|
|                   -10|                       -40| [...]
{'fraction_non_alphanumeric': 0.2787790697674419, 'fraction_numerical': 0.13284883720930232, 'mean_word_length': 1.4826839826839826, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 5, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '343994', 'license': 'MIT-0', 'n_tokens_mistral': 1397, 'n_tokens_neox': 1081, 'n_words': 154, 'path': 'pyspark-tutorial/pyspark-ml-basic.ipynb', 'repo_name': 'samlexrod/sakeoflearning', 'size': '3440'}