<jupyter_start><jupyter_text>Data Scientist Jr.: Dr.<NAME>  PySpark DataFrame* Filter operation* AND = &  , OR = | ,  ==* ~<jupyter_code>from pyspark.sql import SparkSession 
spark = SparkSession.builder.appName('dataframe').getOrCreate()
df_pyspark = spark.read.csv('test2_without_null_values.csv', header=True, inferSchema=True)
df_pyspark.show()<jupyter_output>+---------+---+----------+------+
|     Name|age|Experience|Salary|
+---------+---+----------+------+
|    Krish| 31|        10| 30000|
|Sudhanshu| 30|         8| 25000|
|    Sunny| 29|         4| 20000|
|     Paul| 24|         3| 20000|
|   Harsha| 21|         1| 15000|
|  Shubham| 23|         2| 18000|
+---------+---+----------+------+<jupyter_text>Filter operations<jupyter_code># Salary of the people less than or equal to 20000

df_pyspark.filter('Salary <= 20000').show()
# Selecting only with Name and age

df_pyspark.filter('Salary <= 20000').select(['Name', 'age']).show()
# Another form, also

df_pyspark.filter(df_pyspark['Salary'] <= 20000).show()
# Using the logical operators: AND, OR
# see the parentheses

df_pyspark.filter((df_pyspark['Salary'] <= 20000) & (df_pyspark['Salary'] >= 18000)).show()
df_pyspark.filter((df_pyspark['Salary'] <= 20000) | (df_pyspark['Salary'] >= 18000)).show()
# Using the logical operator: NOT = ~
# OBS ==> Isso faz que não pegue o valor de 20000 mesmo

df_pyspark.filter(~(df_pyspark['Salary'] <= 20000)).show()<jupyter_output>+---------+---+----------+------+
|     Name|age|Experience|Salary|
+---------+---+----------+------+
|    Krish| 31|        10| 30000|
|Sudhanshu| 30|         8| 25000|
+---------+---+----------+------+
{'fraction_non_alphanumeric': 0.25790754257907544, 'fraction_numerical': 0.07116788321167883, 'mean_word_length': 2.7133182844243793, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '13823', 'license': 'MIT', 'n_tokens_mistral': 696, 'n_tokens_neox': 598, 'n_words': 118, 'path': 'PySpark_4.ipynb', 'repo_name': 'EddyGiusepe/PySpark_com_Python', 'size': '1666'}