<jupyter_start><jupyter_text>Load replicate weights using a generator because the file is large<jupyter_code>def load_replicates(filename):
    # build iterator
    itr = pd.read_stata(
        filename,
        preserve_dtypes = True,
        chunksize = 10000,
    )

    df = pd.DataFrame()

    for chunk in itr:
        df = df.append(chunk)
        
    # filter list and add uid
    res = (
        df
        .assign(
            uid = lambda x: x['ssuid'].astype(str) + "-" + x["pnum"].astype(str),
        )
        .loc[ lambda x: x["monthcode"] == 12 ]
        .drop(columns = ["ssuid", "pnum", "spanel", "monthcode"])
    )
    
    return res
w1 = load_replicates("../data/rw14w1.dta")

w1.drop(
    # This column is the same as the "wpfinwgt" column in the public use notebook
    columns = ["repwt0"] 
).to_csv("../output/w1_replicates.csv", index = False)

w4 = load_replicates("../data/rw14w4.dta")

w4.to_csv("../output/w4_replicates.csv", index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14257228315054835, 'fraction_numerical': 0.019940179461615155, 'mean_word_length': 2.5104895104895104, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '417778', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 369, 'n_tokens_neox': 356, 'n_words': 81, 'path': 'notebooks/load_replicates.ipynb', 'repo_name': 'BuzzFeedNews/2019-11-sipp', 'size': '1003'}