<jupyter_start><jupyter_text>Data GeneratorGenerating more realistic data<jupyter_code># helper functions
from datagen_helper import generate_pd_df

import pandas as pd
df_pd = generate_pd_df(20)

df_pd.to_csv('transactions-sample.csv', sep=',', index=False)
print (f'wrote to "transactions-sample.csv"')

df_pd.head(10)<jupyter_output>wrote to "transactions-sample.csv"<jupyter_text>Create a Spark DataFrame<jupyter_code>## Init Spark
import os, sys

this_dir = os.path.abspath('')
parent_dir = os.path.dirname(this_dir)
grand_parent_dir = os.path.dirname(parent_dir)
    
if parent_dir not in sys.path:
    sys.path.append(parent_dir)

from init_spark import init_spark

spark = init_spark()
import pyspark
from pyspark.sql.types import ArrayType, IntegerType, LongType, StringType, FloatType, TimestampType, StructType, StructField

my_schema = StructType([
                       StructField("id", StringType(), True),
                       StructField("timestamp", StringType(), True),
                       StructField("mti", StringType(), True),
                       StructField("card_number", StringType(), True),
                       StructField("amount_customer", FloatType(), True),
                       StructField("merchant_type", StringType(), True),
                       StructField("merchant_id", StringType(), True),
                       StructField("merchant_address", StringType(), True),
                       StructField("ref_id", StringType(), True),
                       StructField("amount_merchant", FloatType(), True),
                       StructField("response_code", StringType(), True),
                      ])


spark_df = spark.createDataFrame(df_pd, schema=my_schema)
spark_df.show(truncate=True)<jupyter_output>+--------------------+--------------------+----+----------------+---------------+-------------+-----------+--------------------+--------------------+---------------+-------------+
|                  id|           timestamp| mti|     card_number|amount_customer|merchant_type|merchant_id|    merchant_address|              ref_id|amount_merchant|response_code|
+--------------------+--------------------+----+----------------+---------------+-------------+-----------+--------------------+--------------------+---------------+-------------+
|1c309b20-5074-41c...|2017-06-21 17:48:...|0100|4789625716349534|           95.0|         0167|         81|Star Hardware, 39...|                    |            NaN|          NaN|
|fd421703-f724-425...|2017-06-21 17:48:...|0110|4789625716349534|           95.0|         0167|         81|Star Hardware, 39...|1c309b20-5074-41c...|          92.15|           00|
|f06b52bc-2944-408...|2017-09-27 14:26:...|0100|4885453044056032|           8.19|         8371|     [...]
{'fraction_non_alphanumeric': 0.23780267437658115, 'fraction_numerical': 0.0657752078062884, 'mean_word_length': 1.7137254901960783, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '489479', 'license': 'Apache-2.0', 'n_tokens_mistral': 893, 'n_tokens_neox': 750, 'n_words': 133, 'path': 'data-generator/datagen-tx-small-1.ipynb', 'repo_name': 'elephantscale/advanced-sql-labs', 'size': '2767'}