<jupyter_start><jupyter_text>Table of Contents<jupyter_code>%matplotlib inline
import pandas as pd
import numpy as np
NAN_IDENTIFIERS = [
    '[-1] keine Angabe', '[-2] trifft nicht zu',
    '[-3] nicht valide',
    '[-5] In Fragebogenversion nicht enthalten',
    '[-8] Frage in diesem Jahr nicht Teil des Frageprograms',
]

VARIABLE_NAMES_PGEN = {
#     General variables
#     'cid': 'ID_ORIGINAL_HH',  # Case id, id of original household
    'syear': 'YEAR',  # survey year
#     'hid': 'ID_HH',  # current household id
    'pid': 'ID',  # permanent personal id
    'pgcasmin': 'EDUCATION_GROUPS_CASMIN',
    'pgisced97': 'EDUCATION_GROUPS_ISCED97',
    'pgisced11': 'EDUCATION_GROUPS_ISCED11',
}

RETAINED_COLUMNS_PGEN = list(VARIABLE_NAMES_PGEN.keys())
# Load dataset
df = pd.read_stata('../data/pgen.dta', columns=RETAINED_COLUMNS_PGEN)
df = df.rename(columns=VARIABLE_NAMES_PGEN)
# Identify missing values
df.replace(to_replace=NAN_IDENTIFIERS, value=np.nan, inplace=True)
for i in [i for i in df if 'EDUCATION' in i]:
    df[i].cat.remove_unused_categories(inplace=True)
df.info()
df.EDUCATION_GROUPS_ISCED97.cat.categories<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12908777969018934, 'fraction_numerical': 0.012908777969018933, 'mean_word_length': 4.964102564102564, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '108460', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 463, 'n_tokens_neox': 462, 'n_words': 98, 'path': 'src/exploration/explore_pgen.ipynb', 'repo_name': 'tobiasraabe/locus-of-control', 'size': '1162'}