<jupyter_start><jupyter_text>Initial EDAIn this notebook we explore the preprocessed data tables<jupyter_code># Load the "autoreload" extension
%load_ext autoreload
%autoreload 2

import sys
import os
import numpy as np
import pandas as pd
import matplotlib
%matplotlib inline
#%matplotlib notebook
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.style.use('ggplot')

# add the 'src' directory as one where we can import modules
src_dir = os.path.abspath(os.path.join(os.getcwd(), os.pardir, 'src'))
sys.path.append(src_dir)

# self written packages
from data import data_utils<jupyter_output><empty_output><jupyter_text>Read files from directory of processed data tables<jupyter_code>dfs = data_utils.read_csv_from_dir(dir_name="processed")
print("We have {} tables".format(len(dfs)))<jupyter_output>We have 11 tables<jupyter_text>Head all 11 tables:<jupyter_code>dfs[0].head()

dfs[1].head()
dfs[2].head()
dfs[3].head()
dfs[4].head()
dfs[5].head()
dfs[6].head()
dfs[7].head()
dfs[8].head()
dfs[9].head()
dfs[10].head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12982126058325494, 'fraction_numerical': 0.01599247412982126, 'mean_word_length': 7.3125, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '258280', 'license': 'MIT', 'n_tokens_mistral': 411, 'n_tokens_neox': 399, 'n_words': 95, 'path': 'notebooks/1.0-lm-initial-EDA.ipynb', 'repo_name': 'Lumik7/bi2_project2', 'size': '1063'}