<jupyter_start><jupyter_text>```bashexport PATH=${SPARK_HOME}/bin:${PATH}export SPARK_LOCAL_IP=localhostexport PYSPARK_DRIVER_PYTHON=jupyterexport PYSPARK_DRIVER_PYTHON_OPTS='lab --ip=0.0.0.0 --allow-root --no-browser --NotebookApp.token=""'export PACKAGES="com.esri:filegdb:0.12.5"export PACKAGES="${PACKAGES},com.esri:spark-functions:0.10"pyspark\  --master local[*]\  --num-executors 1\  --driver-memory 30G\  --executor-memory 30G\  --conf spark.ui.enabled=false\  --packages ${PACKAGES}\  --exclude-packages org.scala-lang:scala-reflect```<jupyter_code>import os
_ = sql("select 'Initialize Spark'").collect()
spark._jvm.com.esri.spark.Functions.registerFunctions()
gdb = spark._jvm.com.esri.gdb.FileGDB
gdb_path = os.path.join("data", "Miami.gdb")
for tab in gdb.listTables(gdb_path):
    print(tab)
df = spark.read \
    .format('com.esri.gdb') \
    .options(path=gdb_path, name='Broadcast') \
    .load()
df.printSchema()
df.count()
df.registerTempTable("bc")
sql("""
select mmsi,dt1,x1,y1,dt2,x2,y2
from (
select MMSI mmsi,
BaseDateTime as dt1,
Shape.x as x1,
Shape.y as y1,
lead(BaseDateTime) over (partition by MMSI order by BaseDateTime) as dt2,
lead(Shape.x) over (partition by MMSI order by BaseDateTime) as x2,
lead(Shape.y) over (partition by MMSI order by BaseDateTime) as y2
from bc
)
where dt1 < dt2 and haversine(x1,y1,x2,y2) > 500.0
""").show(32, truncate=True)<jupyter_output>+---------+--------------------+----------+------------------+--------------------+----------+------------------+
|     mmsi|                 dt1|        x1|                y1|                 dt2|        x2|                y2|
+---------+--------------------+----------+------------------+--------------------+----------+------------------+
|209570000| 2009-01-03 23:30:00|-79.972952|25.910472999999996|2009-01-03 23:31:...| -79.97176|         25.906058|
|209570000|2009-01-03 23:44:...| -79.95738|25.854034999999996| 2009-01-03 23:45:00|-79.956087|25.849666999999997|
|209570000| 2009-01-04 00:15:00|-79.923492|25.737345000000005|2009-01-04 00:16:...|-79.922198|25.732974999999996|
|367136710| 2009-01-02 22:09:00|-79.844383|25.592617000000004|2009-01-02 22:10:...|-79.843567|25.599082999999993|
|367136710|2009-01-02 22:10:...|-79.843567|25.599082999999993|2009-01-02 22:11:...|-79.842817|25.605099999999993|
|367136710|2009-01-02 22:11:...|-79.842817|25.605099999999993| 2009-01-02 22:12:00| -79.[...]
{'fraction_non_alphanumeric': 0.2574875207986689, 'fraction_numerical': 0.21630615640599002, 'mean_word_length': 2.7461059190031154, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 10, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '234449', 'license': 'Apache-2.0', 'n_tokens_mistral': 1277, 'n_tokens_neox': 925, 'n_words': 139, 'path': 'BroadcastHaversine.ipynb', 'repo_name': 'cakilandeswari/FileGDB', 'size': '2404'}