<jupyter_start><jupyter_text>save as csv format<jupyter_code>df_csv = spark.read.csv("../data/ratings.csv", header=True)
df_csv.show()
df_csv.write.
df_csv.write.csv("../output/ratings_1.csv", header=True, mode='overwrite')
!ls -lh ../output/ratings_1.csv/
!ls ../output/ratings_1.csv
!head ../output/ratings_1.csv/part-00001-aece805c-20a7-4225-b152-40316bc8fc5e-c000.csv
df_csv.coalesce(1).write.csv("../output/ratings_coalease_1.csv", header=True)
!ls -lh ../output/ratings_coalease_1.csv/<jupyter_output>total 1017088
-rw-r--r--  1 bryan.yang  staff     0B Nov  6 01:05 _SUCCESS
-rw-r--r--  1 bryan.yang  staff   490M Nov  6 01:05 part-00000-f0bf70a3-4fc6-4afb-9068-948ef3a3ee45-c000.csv<jupyter_text>save as json<jupyter_code>df_csv.write.json("../output/ratings.json", mode='overwrite')
!ls ../output/ratings.json/
!head ../output/ratings.json/part-00002-a7190b28-fc05-4b26-8ece-f9fb95ee867c-c000.json
!ls -lh ../output/ratings.json/
!ls -lh ../output/ratings.csv/<jupyter_output>total 1052032
-rw-r--r--  1 bryan.yang  staff     0B Nov  2 09:37 _SUCCESS
-rw-r--r--  1 bryan.yang  staff   123M Nov  2 09:37 part-00000-91cd7a56-8c74-48f5-ab08-2c79b2b1911e-c000.csv
-rw-r--r--  1 bryan.yang  staff   123M Nov  2 09:37 part-00001-91cd7a56-8c74-48f5-ab08-2c79b2b1911e-c000.csv
-rw-r--r--  1 bryan.yang  staff   123M Nov  2 09:37 part-00002-91cd7a56-8c74-48f5-ab08-2c79b2b1911e-c000.csv
-rw-r--r--  1 bryan.yang  staff   120M Nov  2 09:37 part-00003-91cd7a56-8c74-48f5-ab08-2c79b2b1911e-c000.csv<jupyter_text>save as parquet<jupyter_code>df_csv.write.parquet("../output/ratings.parquet", mode = 'overwrite')
!ls -lh ../output/ratings.parquet/<jupyter_output>total 317440
-rw-r--r--  1 bryan.yang  staff     0B Nov  6 01:11 _SUCCESS
-rw-r--r--  1 bryan.yang  staff    40M Nov  6 01:11 part-00000-c53bcf1a-b4a0-4d1c-8a26-5a613000ae36-c000.snappy.parquet
-rw-r--r--  1 bryan.yang  staff    39M Nov  6 01:11 part-00001-c53bcf1a-b4a0-4d1c-8a26-5a613000ae36-c000.snappy.parquet
-rw-r--r--  1 bryan.yang  staff    39M Nov  6 01:11 part-00002-c53bcf1a-b4a0-4d1c-8a26-5a613000ae36-c000.snappy.parquet
-rw-r--r--  1 bryan.yang  staff    37M Nov  6 01:11 part-00003-c53bcf1a-b4a0-4d1c-8a26-5a613000ae36-c000.snappy.parquet<jupyter_text>Compressionbzip2, gzip, lz4, snappy and deflate<jupyter_code>df_csv.write.csv("../output/ratings_gzip.csv", header=True, compression='gzip', mode='overwrite')
!ls -lh ../output/ratings_gzip.csv<jupyter_output>total 262552
-rw-r--r--  1 bryan.yang  staff     0B Nov  6 01:13 _SUCCESS
-rw-r--r--  1 bryan.yang  staff    33M Nov  6 01:13 part-00000-d2151638-7197-4ca8-b523-d1f339da1d84-c000.csv.gz
-rw-r--r--  1 bryan.yang  staff    32M Nov  6 01:13 part-00001-d2151638-7197-4ca8-b523-d1f339da1d84-c000.csv.gz
-rw-r--r--  1 bryan.yang  staff    32M Nov  6 01:13 part-00002-d2151638-7197-4ca8-b523-d1f339da1d84-c000.csv.gz
-rw-r--r--  1 bryan.yang  staff    30M Nov  6 01:13 part-00003-d2151638-7197-4ca8-b523-d1f339da1d84-c000.csv.gz<jupyter_text>存成 table<jupyter_code>spark.sql("show tables").show()
df_csv.write.saveAsTable('ratings_csv')
spark.sql("show tables").show()
spark.sql("select * from ratings_csv").show()<jupyter_output>+------+-------+------+----------+
|userId|movieId|rating| timestamp|
+------+-------+------+----------+
|     1|      2|   3.5|1112486027|
|     1|     29|   3.5|1112484676|
|     1|     32|   3.5|1112484819|
|     1|     47|   3.5|1112484727|
|     1|     50|   3.5|1112484580|
|     1|    112|   3.5|1094785740|
|     1|    151|   4.0|1094785734|
|     1|    223|   4.0|1112485573|
|     1|    253|   4.0|1112484940|
|     1|    260|   4.0|1112484826|
|     1|    293|   4.0|1112484703|
|     1|    296|   4.0|1112484767|
|     1|    318|   4.0|1112484798|
|     1|    337|   3.5|1094785709|
|     1|    367|   3.5|1112485980|
|     1|    541|   4.0|1112484603|
|     1|    589|   3.5|1112485557|
|     1|    593|   3.5|1112484661|
|     1|    653|   3.0|1094785691|
|     1|    919|   3.5|1094785621|
+------+-------+------+----------+
only showing top 20 rows
{'fraction_non_alphanumeric': 0.18181818181818182, 'fraction_numerical': 0.2269011642308645, 'mean_word_length': 2.7148114075436984, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '423500', 'license': 'Apache-2.0', 'n_tokens_mistral': 2439, 'n_tokens_neox': 1943, 'n_words': 286, 'path': 'pyspark/example/spark_sql/5.4_savedata.ipynb', 'repo_name': 'chiliangpi/hellobi', 'size': '4037'}