def parse_dt():

    def _parse(dt):
        return datetime.strptime(dt, '%d-%b-%y %I.%M.%S.%f %p')
return udf(_parse, TimestampType())
df_with_timestamp = df.withColumn('timestamp', parse_dt()(df.datetime))

def in_last_5_minutes(now):

    def _in_last_5_minutes(then):
        return then > now - timedelta(minutes=5)
return udf(_in_last_5_minutes, BooleanType())
df_with_timestamp.where(in_last_5_minutes(now)(col('timestamp')))
Фильтрация столбцов в PySpark