from pyspark.sql.types import IntegerType
from pyspark.sql.functions import udf
size_ = udf(lambda xs: len(xs), IntegerType())
df.where(size_(col('tokens')) <= 3).show()
Filtering DataFrame using the length of a column