<jupyter_start><jupyter_text>Import Data  JSON Dataset<jupyter_code>dataf = df()#pd.read_json("db.json")<jupyter_output><empty_output><jupyter_text>Excel Dataset<jupyter_code>df1 = pd.read_excel("../datasets/raw_data.xlsx", sheetname='Sheet1')
data = df1.fillna(method='ffill')<jupyter_output><empty_output><jupyter_text>HTML Dataset<jupyter_code>dataf = df()#pd.read_html(‘linktoonlinehtmlfile’)<jupyter_output><empty_output><jupyter_text>XML Dataset<jupyter_code>dataf = df() #to be completed<jupyter_output><empty_output><jupyter_text>CSV Dataset<jupyter_code>dataf = pd.read_csv("../../data/raw/eip/disease.csv", sep=' ', delimiter='\t')
import pandas_profiling

profile = pandas_profiling.ProfileReport(dataf)
profile.to_file(outputfile="output.html")<jupyter_output>/usr/local/lib/python3.7/site-packages/matplotlib/font_manager.py:1241: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans.
  (prop.get_family(), self.defaultFamily[fontext]))
/usr/local/lib/python3.7/site-packages/matplotlib/font_manager.py:1241: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans.
  (prop.get_family(), self.defaultFamily[fontext]))
/usr/local/lib/python3.7/site-packages/matplotlib/font_manager.py:1241: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans.
  (prop.get_family(), self.defaultFamily[fontext]))<jupyter_text>Unit tests<jupyter_code>def test_import_data():
    assert os.path.exists("../datasets/disease.csv") == 1
    assert os.path.exists("../datasets/raw_data.xlsx") == 1
    assert pd.read_csv("../datasets/disease.csv", sep=' ', delimiter='\t').size != 0
    assert pd.read_excel("../datasets/raw_data.xlsx", sheetname='Sheet1').size != 0
ipytest.run()<jupyter_output>=============================== test session starts ===============================
platform linux -- Python 3.6.7, pytest-4.3.1, py-1.8.0, pluggy-0.9.0
rootdir: /home/tgey/Medicision/IA/Notebooks, inifile:
collected 4 items

Data_visualization.py ....                                                  [100%]

============================ 4 passed in 0.38 seconds =============================<jupyter_text>Data Exploration  Dataset discover<jupyter_code>dataf.info()
dataf.columns
dataf.columns=['Symptoms', 'Diseases', 'occurrence', 'score']
dataf.head(15)<jupyter_output><empty_output><jupyter_text>Missing Values<jupyter_code>pd.isnull(dataf).any()
total = dataf.isnull().sum().sort_values(ascending=False)
percent = (dataf.isnull().sum()/dataf.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
missing_data.head(20)
def test_missing_values():
    assert missing_data['Total'].max() == 0

ipytest.run()
dataf.describe()
nbSymptoms = dataf.groupby("Diseases")[["Symptoms"]].count()
nbSymptoms.sort_values("Symptoms").reset_index()
nbSymptoms.columns=['Number of Symptoms for a disease']
nbSymptoms.head(15)
def test_symptoms_agregation():
    assert ('Number of Symptoms for a disease' in nbSymptoms.columns) == True
    assert nbSymptoms.size != 0
nbDiseases = dataf.groupby("Symptoms")[["Diseases"]].count()
nbDiseases.sort_values("Diseases").reset_index()
nbDiseases.columns=['Number of Diseases']
nbDiseases.head(15)
def test_diseases_agregation():
    assert ('Number of Diseases' in nbDiseases.columns) == True
    assert nbDiseases.size != 0<jupyter_output><empty_output><jupyter_text>Data Visualization  Feature Correlation<jupyter_code>from pandas.plotting import scatter_matrix
attributes = ["score", "occurrence"]
scatter_matrix(dataf[attributes], figsize=(12, 8))
corrmat = dataf.corr()
corrmat['score'].sort_values(ascending=False)
f, ax = plt.subplots(figsize=(12, 9))
sns.heatmap(corrmat, vmax=.8, square=True);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1837908496732026, 'fraction_numerical': 0.01699346405228758, 'mean_word_length': 6.652, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '409195', 'license': 'MIT', 'n_tokens_mistral': 1429, 'n_tokens_neox': 1366, 'n_words': 206, 'path': 'jupyter/notebooks/eda/Data_visualization.ipynb', 'repo_name': 'tgey/HubNotebooks', 'size': '3825'}