<jupyter_start><jupyter_text>Dask DataFrame Experiment Part 1Get fake data from mimesis for the dataframes that will be used in the experiment.  Imports<jupyter_code># ! pip install mimesis
from mimesis import Person, Address, Datetime
from mimesis.enums import Gender

import os
import math
import time
import pickle
import random
import pandas as pd

from itertools import accumulate

# %config IPCompleter.greedy=True<jupyter_output><empty_output><jupyter_text>Create mockup data and accompanying layouts<jupyter_code>person = Person('en')
address = Address('en')
datetime = Datetime()<jupyter_output><empty_output><jupyter_text>Functions to generate the data<jupyter_code>def generate_person():
    return {
        'rec': 'PRSN',
        'id': random.randint(10000, 100000000),
        'first_name': person.first_name(),
        'last_name': person.last_name(),
        'gender': person.gender(),
        'ssn': random.randint(1000000, 100000000),
    }

def generate_address(id, ssn):
    return {
        'rec': 'ADDR',
        'id': id,
        'ssn': ssn,
        'address': address.address(),
        'city': address.city(),
        'state': address.state(),
        'country': 'USA',
        'begin': datetime.date(start=2000, end=2019)
    }

def generate_email(id, ssn):
    return {
        'rec': 'EMAL',
        'id': id,
        'ssn': ssn,
        'email': person.email(),
        'begin': datetime.date(start=2000, end=2019)
    }

def create_accounts(num=1):
    output = []
    for _a in range(num):
        p = generate_person()
        output.append(p)
        for _b in range(random.randint(1, 5)):
            output.append(generate_address(p['id'], p['ssn']))
        for _c in range(random.randint(1, 5)):
            output.append(generate_email(p['id'], p['ssn']))
    return output<jupyter_output><empty_output><jupyter_text>Functions to create the layout from the generated data<jupyter_code># For padding length of variables to provide 'space' in the final layout
def round_up_5(num):
    return int(math.ceil(num/5)*5)

# Get the length of each column that will fit the max length values and pad for some extra width
def create_lengths(df):
    d = {}
    for col in df.columns:
        try:
            length = df[df[col].notnull()][col].str.len().max()
            length = round_up_5(int(length*1.5))
        except:
            if df[col].dtype == 'O':
                length = 10
            else:
                length = 15
        d[col] = length
    return d

# Convert a record to a layout using the field length information
def layout_from_record(d: dict, length_d: dict):
    length = [length_d[k] for k in d.keys()]
    
    return {
        'rec': ["DTL"+d['rec'] for _ in range(len(d.keys()))],
        'name': list(d.keys()),
        'start': [x+1 for x in [0]+list(accumulate(length[:-1]))],
        'length': length,
        'conversion': ['CHAR' for _ in range(len(d.keys()))],
    }

# Convert a data dataframe to a layout
def convert_data_to_layout(df):
    # Get the lengths of the data
    length_d = create_lengths(df)

    # Get the layouts for the 3 records
    layout_df = pd.concat([
        pd.DataFrame(layout_from_record(generate_person(), length_d)),
        pd.DataFrame(layout_from_record(generate_address(1, 2), length_d)),
        pd.DataFrame(layout_from_record(generate_email(1, 2), length_d)),
    ])

    # Make sure the records are sorted correctly
    layout_df.sort_values(by=['rec', 'start'], inplace=True)

    return layout_df<jupyter_output><empty_output><jupyter_text>Function to create a record definition tableNote -- Based on experience, we 'know' that the "rec" column is the column that defines the layout of the dataIdeally, this file specification would be stored in a database or other documentation repository.<jupyter_code># Convert a data dataframe to a layout
def convert_data_to_record_definitions(df):

    # Unique 'rec' values
    records = df['rec'].unique().tolist()

    # Construct the dataframe definitions from the record 'rec' values
    definitions = ["data_df['rec']=='" + rec + "'" for rec in records]

    # Construct the record labels from the record 'rec' values
    records = ["DTL"+rec for rec in records]

    # Zip the 2 lists together into a 'records' definition dataframe
    records_df = pd.DataFrame(zip(records, definitions), columns=['Record', 'Definition'])
    return records_df<jupyter_output><empty_output><jupyter_text>Generate the dataGenerate sets of data starting at 5000 and increasing by an order of magnitude.<jupyter_code># Data location
DATA = 'data'

def generate_data(count=5000):
    file_path_data = os.path.join(os.getcwd(), DATA, str(count)+'_data.pickle')
    file_path_layout = os.path.join(os.getcwd(), DATA, str(count)+'_layout.pickle')
    file_path_records = os.path.join(os.getcwd(), DATA, str(count)+'_records.pickle')

    data_df = pd.DataFrame(create_accounts(count))
    layout_df = convert_data_to_layout(data_df)
    records_df = convert_data_to_record_definitions(data_df)
    
    with open(file_path_data, 'wb') as f:
        pickle.dump(data_df, f)
    with open(file_path_layout, 'wb') as f:
        pickle.dump(layout_df, f)
    with open(file_path_records, 'wb') as f:
        pickle.dump(records_df, f)
    
%%time
# generate_data(5000)
generate_data(50000)
# generate_data(500000)
# generate_data(5000000)<jupyter_output>CPU times: user 7.06 s, sys: 146 ms, total: 7.2 s
Wall time: 7.26 s<jupyter_text>Time taken to create the dataUsing TR1950 Ubuntu1804 with 16GB RAM in docker container- 5,000 person accounts takes approx. 0.7 seconds- 50,000 person accounts takes approx. 7 seconds- 500,000 person accounts takes approx. 72 seconds- 5,000,000 person accounts results is kernel shutdown likely due to RAM limitUsing i7-7700HQ Win10 with 16GB RAM- 5,000 person accounts takes approx. 0.5 seconds --> 34,932 records- 50,000 person accounts takes approx. 5 seconds --> 350,196 records- 500,000 person accounts takes approx. 57 seconds --> 3,500,295 records- 5,000,000 person accounts takes approx. 1800* seconds --> 34,999,259 recordsNote - The generation of 5 million person accounts resulted in a RAM limit that slowed down the generation of data.<jupyter_code>os.listdir(os.path.join(os.getcwd(), DATA))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11684394348309256, 'fraction_numerical': 0.03429115732655977, 'mean_word_length': 3.670126019273536, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '523690', 'license': 'MIT', 'n_tokens_mistral': 2144, 'n_tokens_neox': 1967, 'n_words': 588, 'path': 'Part 1 - Get Data.ipynb', 'repo_name': 'daqieq/dask_dataframe_experiment', 'size': '6299'}