<jupyter_start><jupyter_text>Generating synthetic data<jupyter_code>import pandas as pd 
import numpy as np 
from numpy.random import normal
from numpy.random import exponential
from numpy.random import choice
import string
from random import choice as ch 
# number of points to use: 
n_points = 200<jupyter_output><empty_output><jupyter_text>Generating age * not normally distrubuted    * 5% nulls    * 3% ambiguous outliers    * 5% clear outliers (outside of p01, p99)<jupyter_code># generating an exponential distribution
ages = pd.Series([exponential(15) for x in range(n_points)]) + 18

# Generating 5% nulls 
fraction = .05
ages.loc[ages.sample(frac=fraction).index] = np.nan

# Made only .5% "super-outlier", because 5% would affect the means a lot
fraction = .005
ages.loc[ages.sample(frac=fraction).index] = [normal(180, 50) for x in range(len(ages.sample(frac=fraction)))]

# making 1% super-super outlier, to mess up the standard deviation and mean 
ages.loc[50] = 300000000

# Making 4% outliers under 18 
fraction = .04
ages.loc[ages.sample(frac=fraction).index] = normal(5, 3)

# Choosing only positives
ages = ages[ages>0]

# Making them integers
ages = ages.astype(int)<jupyter_output><empty_output><jupyter_text>Generating the categorical data * gender    * 40% 'male'    * 55% 'female'    * 2% 'MALE'    * 3% null<jupyter_code>gender = pd.Series([choice(['male', 'female', 'MALE', np.NaN], 
                           p=[.4, .55, .02, .03]) for x in range(n_points)])<jupyter_output><empty_output><jupyter_text>Generating the height data * normally distributed    * 3% 2-sigma outliers    * 2% 3-sigma outliers    * 2% missing data<jupyter_code>heights = pd.Series([normal(170, 15) for x in range(n_points)])

# make a couple of super outliers 
heights.iloc[20] = 252
heights.iloc[21] = 65
heights.iloc[22] = 235

# force to integers: 
heights = heights.astype(int)

# make 2% missing data
fraction = .02
heights.loc[heights.sample(frac=fraction).index] = np.NaN<jupyter_output><empty_output><jupyter_text>Joining them together:<jupyter_code>all_data = pd.DataFrame([ages, heights, gender]).T
all_data.columns = ['age', 'height', 'gender']<jupyter_output><empty_output><jupyter_text>Preview:<jupyter_code>all_data.head()<jupyter_output><empty_output><jupyter_text>Generating some duplicated data:<jupyter_code>entry = all_data.iloc[4] 
entry<jupyter_output><empty_output><jupyter_text>Making an index:<jupyter_code>n = 10
index = ["".join(ch(string.ascii_uppercase) for i in range(n)) for x in range(n_points)]

# forcing the index of the duplicated data to be duplicated too: 
for row in range(4,8): 
    all_data.iloc[row] = entry  
    index[row] = 'YPUQAPSOYJ'

all_data.index = index
all_data.head(20)
all_data.to_csv('all_data.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1252692031586504, 'fraction_numerical': 0.03374012921751615, 'mean_word_length': 5.319727891156463, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '161115', 'license': 'MIT', 'n_tokens_mistral': 1094, 'n_tokens_neox': 974, 'n_words': 245, 'path': 'course/class2/data/making-synthetic-data.ipynb', 'repo_name': 'AlexGomesDS/data-science-101', 'size': '2786'}