<jupyter_start><jupyter_text>Initializing PySpark<jupyter_code>#intialize Spark Context
#from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
spark= SparkSession.builder.appName('Quanitative').getOrCreate()
spark<jupyter_output><empty_output><jupyter_text>importing Libraries<jupyter_code>from pyspark.sql import *
from pyspark.sql.types import *
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
intern=pd.read_csv('/Users/Asus/Documents/InternCsv/InternCsvfinal_dispatch(Act).csv')  
#segregating excel files from the Given Data
intern.shape
intern.columns
intern.fillna(0,inplace=True)
intern = intern.rename(columns={'Unnamed: 0': 'Index'}, index={'ONE': 'Row_1'})
intern.head(4)
intern.info()
intern.isnull().sum()
intern["Plant"].value_counts()
# From graph we can see that Plants Dispatches in July

# Plant vs Dispatch
sns.catplot(y = "Dispatch", x = "Plant", data = intern.sort_values("Dispatch", ascending = False), kind="boxen", height = 6, aspect = 4)
plt.show(10)
# Time vs Dispatch

sns.catplot(y = "Dispatch", x = "Time", data = intern.sort_values("Dispatch", ascending = False), kind="boxen", height = 4, aspect = 3)
plt.show()
# Finds correlation between Independent and dependent attributes

plt.figure(figsize = (18,18))
sns.heatmap(intern.corr(), annot = True, cmap = "RdYlGn")

plt.show()
X = intern.loc[:, 'Date']
X.head(4)
y = intern.iloc[:,4]
y.head(4)
from datetime import datetime
intern['Date'] = pd.to_datetime(intern['Date']).dt.date
intern.dtypes

intern['Date'] = pd.to_datetime(intern['Date'])
from datetime import datetime
intern.dtypes
df_spark = spark.createDataFrame(intern)
df_spark.show()<jupyter_output>+-----+--------------+-------------------+----+--------+
|Index|         Plant|               Date|Time|Dispatch|
+-----+--------------+-------------------+----+--------+
|    0|        Canyon|2020-07-01 00:00:00|0:00|      20|
|    1|           WPS|2020-07-01 00:00:00|0:00|       0|
|    2|          Olax|2020-07-01 00:00:00|0:00|      10|
|    3|          Nlax|2020-07-01 00:00:00|0:00|      39|
|    4|          Polp|2020-07-01 00:00:00|0:00|      39|
|    5|Upper Kothmale|2020-07-01 00:00:00|0:00|       0|
|    6|          Koth|2020-07-01 00:00:00|0:00|      87|
|    7|          Vict|2020-07-01 00:00:00|0:00|       0|
|    8|          Rand|2020-07-01 00:00:00|0:00|       0|
|    9|          Rant|2020-07-01 00:00:00|0:00|       0|
|   10|           Uku|2020-07-01 00:00:00|0:00|      72|
|   11|          Bowa|2020-07-01 00:00:00|0:00|       0|
|   12|           Sam|2020-07-01 00:00:00|0:00|       0|
|   13|        Kukule|2020-07-01 00:00:00|0:00|     104|
|   14|        LVPS 1|2020-07-0[...]
{'fraction_non_alphanumeric': 0.19134897360703812, 'fraction_numerical': 0.11180351906158358, 'mean_word_length': 2.920977011494253, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '330294', 'license': 'Apache-2.0', 'n_tokens_mistral': 1204, 'n_tokens_neox': 1017, 'n_words': 206, 'path': 'Data Analysis/Quantitative & Qualitative Analysis.ipynb', 'repo_name': 'Pandula1234/Data-Analysis-Intern', 'size': '2728'}