<jupyter_start><jupyter_text>Import the python modules<jupyter_code>import numpy as np
import pandas as pd
# Setting working directory.....................................................................
%cd C:/Users/Ujjwa/OneDrive/Desktop/BDA/Semester 1/PDS-Project-main/PDS-Project-main/data<jupyter_output>C:\Users\Ujjwa\OneDrive\Desktop\BDA\Semester 1\PDS-Project-main\PDS-Project-main\data<jupyter_text>Read the data files<jupyter_code>fed1 = pd.read_csv("DEF survey (Responses) - Form Responses 1.csv", header = 0,\
                  names = ['ts','age','sex','work','phy_ff','phy_health','phy_bw', 'phy_ex', 'meal',
              'height','weight','exercise','fruit','veg','cook','spend','income','gymtime',
              'disease','review','rate'])
fed2 = pd.read_csv("DEF2 (Responses) - Form Responses 1.csv", header = 0, \
                  names = ['ts','age','sex','work','phy_ff','phy_health','phy_bw', 'phy_ex', 'meal',
              'height','weight','exercise','fruit','veg','cook','spend','income','gymtime',
              'disease','review','rate'])
fed1.head()
fed2.head()
print(fed1.shape, fed2.shape)<jupyter_output>(45, 21) (213, 21)<jupyter_text>Concate two files<jupyter_code>fed = pd.concat([fed1,fed2])
fed.shape
fed.dtypes<jupyter_output><empty_output><jupyter_text>Drop the unnessery columns<jupyter_code>fed.columns
fed1=fed.drop(labels=['review', 'ts','disease'],axis=1)<jupyter_output><empty_output><jupyter_text>Checking for NA values<jupyter_code>fed1.isnull().values.any()

# We Don't have any NA Values<jupyter_output><empty_output><jupyter_text>Descripive statistics of the data<jupyter_code>fed1.describe()<jupyter_output><empty_output><jupyter_text>Saving the dataframe as csv<jupyter_code>fed1.to_csv('clean-data.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2088888888888889, 'fraction_numerical': 0.01611111111111111, 'mean_word_length': 4.847402597402597, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '164800', 'license': 'MIT', 'n_tokens_mistral': 674, 'n_tokens_neox': 628, 'n_words': 84, 'path': 'python/data_cleaning.ipynb', 'repo_name': 'U77w41/PDS-Project', 'size': '1800'}