<jupyter_start><jupyter_text>**PySpark no Google Colab**--- Configurando o Google Colab para habilitar o uso do PySpark<jupyter_code># Instala o Java JDK 8
!apt-get install openjdk-8-jdk-headless -qq > /dev/null
# Download do Apache Spark 3.1.2
!wget -q https://downloads.apache.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz
# Descompacta o Apache Spark 3.1.2
!tar xf spark-3.1.2-bin-hadoop3.2.tgz
# Remove o arquivo compactado do Apache Spark 3.1.2
!rm -rf spark-3.1.2-bin-hadoop3.2.tgz
# Instala os módulos FindSpark e PySpark
!pip install -q findspark
!pip install -q pyspark<jupyter_output>[K     |████████████████████████████████| 281.3 MB 41 kB/s 
[K     |████████████████████████████████| 198 kB 38.0 MB/s 
[?25h  Building wheel for pyspark (setup.py) ... [?25l[?25hdone<jupyter_text>Configurando o ambiente para uso do PySpark<jupyter_code># Importa os módulos
import os
import findspark
from pyspark.sql import SparkSession
from pyspark.sql import functions as F
#  Define as variáveis ambientes Home do Java e Spark
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = "/content/spark-3.1.2-bin-hadoop3.2"
# Inicia o FindSpark e cria a instância da sessão Spark
findspark.init()
spark = SparkSession.builder.master("local[*]").getOrCreate()<jupyter_output><empty_output><jupyter_text>PySpark pronto para uso, divirta-se!<jupyter_code>dataset = spark.read.format("json") \
  .option("multiLine",True) \
  .load("sample_data/anscombe.json")
dataset.columns
dataset.show(10)
dataset.printSchema()
dataset_agrupado = dataset \
  .groupBy("Series") \
  .agg(F.avg("X").alias("X_agrupado")
        , F.avg("Y").alias("Y_agrupado")) \
  .orderBy("Series")<jupyter_output><empty_output><jupyter_text>dataset_agrupado.show()<jupyter_code>dataset_agrupado.explain()
df = spark.createDataFrame(
    [ (1., 4.)
    , (2., 5.)
    , (3., 6.)]
    , ["A", "B"])
df.show()
df = spark.createDataFrame(
  [
      ('864.754.453-33,565.878.787-43',)
    , ('565.878.787-43 864.754.453-33 565.878.787-43',)
    , ('333.444.555-66 222.222.222-33',)
  ]
  , ["cpf",])
df.show(10,False)
df.select(
      df.cpf
    , F.length(F.regexp_replace(df.cpf, r'\d+\.\d+\.\d+\-\d+', '')).alias('reg1')
    , F.length(F.regexp_replace(df.cpf, r'\d{3}\.\d{3}\.\d{3}\-\d{2}', '')).alias('reg2')
    , df.cpf.rlike(r'\d{3}\.\d{3}\.\d{3}\-\d{2}').alias('reg3')
).show(10, False)
df2 = spark.createDataFrame(
  [
      ('http://site.com:8080/',)
    , ('http://localhost.com:8080/?pub=200',)
    , ('http://server.com:1234',)
    , ('http://server.com',)
  ]
  , ["url",])
df2.show(10,False)
df2.select(
      df2.url
    , df2.url.rlike(r'https?:[\/]{2}\s+').alias('reg1')
    , df2.url.rlike(r'https?:[\/]{2}([a-zA-Z0-9]+\.[a-zA-Z]{2,4})(:[0-9]+)?').alias('reg2')
    , df2.url.rlike(r'https?:[\/]{2}([a-zA-Z0-9]+\.[a-zA-Z]{2,4})(:[0-9]+)').alias('reg3')
    , df2.url.rlike(r'https?:\/{2}').alias('reg4')
).show(10, False)<jupyter_output>+----------------------------------+-----+----+-----+----+
|url                               |reg1 |reg2|reg3 |reg4|
+----------------------------------+-----+----+-----+----+
|http://site.com:8080/             |false|true|true |true|
|http://localhost.com:8080/?pub=200|false|true|true |true|
|http://server.com:1234            |false|true|true |true|
|http://server.com                 |false|true|false|true|
+----------------------------------+-----+----+-----+----+
{'fraction_non_alphanumeric': 0.273972602739726, 'fraction_numerical': 0.06441270766540368, 'mean_word_length': 2.9814385150812064, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '10347', 'license': 'MIT', 'n_tokens_mistral': 1586, 'n_tokens_neox': 1427, 'n_words': 209, 'path': 'pyspark_google_colab.ipynb', 'repo_name': 'leandrosacramento/PySpark', 'size': '3431'}