<jupyter_start><jupyter_text>OverviewThis notebook will show you how to create and query a table or DataFrame that you uploaded to DBFS. [DBFS](https://docs.databricks.com/user-guide/dbfs-databricks-file-system.html) is a Databricks File System that allows you to store data for querying inside of Databricks. This notebook assumes that you have a file already inside of DBFS that you would like to read from.This notebook is written in **Python** so the default cell type is Python. However, you can use different languages by using the `%LANGUAGE` syntax. Python, Scala, SQL, and R are all supported.<jupyter_code>sc = spark.sparkContext

# File location and type
file_location = "/FileStore/tables/Moby_Dick.txt"
file_type = "txt"

# CSV options
infer_schema = "false"
first_row_is_header = "false"
delimiter = " "

# The applied options are for CSV files. For other file types, these will be ignored.
# df = spark.read.format(file_type) \
#   .option("inferSchema", infer_schema) \
#   .option("header", first_row_is_header) \
#   .option("sep", delimiter) \
#   .load(file_location)

rdd = sc.textFile(file_location)

rdd.collect()
import re

def replace_n_mark(str):

    replace_char = ['.', '"', '-', ',', '*', '(', ')', ';', '[', ']']
    str = re.sub(f"{replace_char}", '', str)
    
    return (str.lower(), 1)
words = rdd.flatMap(lambda x: x.split(' ')) \
            .filter(lambda x: x != '') \
            .map(replace_n_mark) \
            .reduceByKey(lambda x, y: x + y)
words.collect()
# write to file
cols = ['word', 'count']
df = words.toDF(cols)
df.show()
df.createOrReplaceTempView('moby_dick')
%sql

select * from moby_dick
where count > 1000
%sql

select * from moby_dick
where word in ('a', 'an', 'the', 'of', 'he', 'she')
sort by count asc
# Save file to HDFS
df.write.format("parquet") \
        .mode("overwrite") \
        .saveAsTable('moby_dick_csv')
# to delete a file from DBFS, uncomment the below code and provide path

path = '/FileStore/tables/appl_stock-1.csv'
# dbutils.fs.rm(path, True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14000972289742344, 'fraction_numerical': 0.002916869227029655, 'mean_word_length': 3.911694510739857, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '551879', 'license': 'MIT', 'n_tokens_mistral': 714, 'n_tokens_neox': 689, 'n_words': 212, 'path': 'work/pyspark/Text Analysis.ipynb', 'repo_name': 'smandal047/pyspark-pandas', 'size': '2057'}