<jupyter_start><jupyter_text>StarAI - PreprocessThis notebook summarizes the preprocessing of the StarAI datasets. Cf. https://github.com/UCLA-StarAI/Density-Estimation-Datasets  Preliminaries<jupyter_code># Black Codeformatter
%load_ext lab_black<jupyter_output><empty_output><jupyter_text>Constants<jupyter_code>N_JOBS = 4  # Cores on your current CPU, will speed up the process.
RANDOM_STATE = 42<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>import pandas as pd

# import os
import numpy as np
import warnings
from pathlib import Path

from joblib import Parallel, delayed
import missmercs

from missmercs.exps import (
    starai_original_filepath,
    dataset_filepath,
    get_starai_dataset_names,
)<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def load_raw_starai_dfs(name="nltcs"):
    # Load
    fp_train = starai_original_filepath(name=name, kind="train")
    fp_test = starai_original_filepath(name=name, kind="test")

    df_train = pd.read_csv(fp_train, header=None)
    df_test = pd.read_csv(fp_test, header=None)

    # If test is larger than train, we swap them!
    n_rows_train = df_train.shape[0]
    n_rows_test = df_test.shape[0]
    if n_rows_test > n_rows_train:
        warnings.warn(
            "Test set larger than training set. We assume this is a mistake and we swap them."
        )
        df_temp = df_test
        df_test = df_train
        df_train = df_temp
    return df_train, df_test


def drop_constant_columns(df_train, df_test):
    constant_columns = _detect_constant_columns([df_train, df_test])
    df_train, df_test = (
        df_train.drop(constant_columns, axis=1),
        df_test.drop(constant_columns, axis=1),
    )
    return df_train, df_test


def _detect_constant_columns(dfs, constant_column_uvalues=1):
    """Ugly, but works"""
    try:
        result = []
        for df in iter(dfs):
            result += _detect_constant_columns(df)
        return result
    except TypeError:
        # We assume a single dataframe was passed
        return [col for col in dfs if dfs[col].nunique() <= constant_column_uvalues]


def add_headers(df_train, df_test):
    pxs_headers = _headers_pxs(df_train)
    assert pxs_headers == _headers_pxs(
        df_test
    ), "Headers for train and test set differ. That is not supposed to happen!"

    df_train.columns, df_test.columns = pxs_headers, pxs_headers
    return df_train, df_test


def _headers_pxs(df):
    return ["att_{}".format(x) for x in range(len(df.columns))]


def save_starai_dfs(df_train, df_test, name="nltcs", step=1):
    fp_train = dataset_filepath(name=name, step=step, kind="train")
    fp_test = dataset_filepath(name=name, step=step, kind="test")

    df_train.to_csv(fp_train, index=False)
    df_test.to_csv(fp_test, index=False)
    return True
def workfow_raw_to_s01(name, verbose=True):
    if verbose:
        msg = "Start on dataset {}".format(name)
        print(msg)

    df_train, df_test = load_raw_starai_dfs(name=name)
    df_train, df_test = drop_constant_columns(df_train, df_test)
    df_train, df_test = add_headers(df_train, df_test)

    if verbose:
        msg = "Done with dataset {}".format(name)
        print(msg)

    return save_starai_dfs(df_train, df_test, name=name, step=1)<jupyter_output><empty_output><jupyter_text>Get Data<jupyter_code>%%bash
rm -rf ../../data/raw/datasets-starai
git clone <EMAIL>:UCLA-StarAI/Density-Estimation-Datasets.git ../../data/raw/datasets-starai
%%bash
cd ../../data/raw/datasets-starai
git checkout 44c51c50c43686c889de21dcd68cb80820abc9b9<jupyter_output>Note: checking out '<KEY>'.

You are in 'detached HEAD' state. You can look around, make experimental
changes and commit them, and you can discard any commits you make in this
state without impacting any branches by performing another checkout.

If you want to create a new branch to retain commits you create, you may
do so (now or later) by using -b with the checkout command again. Example:

  git checkout -b <new-branch-name>

HEAD is now at 44c51c5 Merge pull request #1 from arranger1044/master<jupyter_text>Preprocess: Raw -> Step 01This is currently the only step we need. If in the future more preprocessing is necessary, this can simply be added as the next step. In that way, the layout and the logic is flexible.  DemoTest<jupyter_code>nltcs_filepath = starai_original_filepath(name="nltcs", kind="train")
print(nltcs_filepath)
assert nltcs_filepath.exists(), "This path ({}) does not exist".format(nltcs_filepath)
workfow_raw_to_s01(name="nltcs")<jupyter_output>Start on dataset nltcs
Done with dataset nltcs<jupyter_text>Preprocess all datasetsRepeat the procedure for all datasets at once.<jupyter_code>starai_dataset_names = get_starai_dataset_names()
starai_dataset_names
Parallel(n_jobs=N_JOBS, verbose=51)(delayed(workfow_raw_to_s01)(ds) for ds in starai_dataset_names)

print("Preprocessing StarAI done")<jupyter_output>[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.
[Parallel(n_jobs=4)]: Done   1 tasks      | elapsed:    2.9s
[Parallel(n_jobs=4)]: Done   2 tasks      | elapsed:    3.0s
[Parallel(n_jobs=4)]: Done   3 tasks      | elapsed:    3.8s
[Parallel(n_jobs=4)]: Done   4 tasks      | elapsed:    3.9s
[Parallel(n_jobs=4)]: Done   5 tasks      | elapsed:    4.4s
[Parallel(n_jobs=4)]: Done   6 tasks      | elapsed:    4.8s
[Parallel(n_jobs=4)]: Done   7 tasks      | elapsed:    5.9s
[Parallel(n_jobs=4)]: Done   8 tasks      | elapsed:    7.6s
[Parallel(n_jobs=4)]: Done   9 tasks      | elapsed:    7.8s
[Parallel(n_jobs=4)]: Done  10 tasks      | elapsed:    7.9s
[Parallel(n_jobs=4)]: Done  11 tasks      | elapsed:    8.1s
[Parallel(n_jobs=4)]: Done  12 tasks      | elapsed:    8.4s
[Parallel(n_jobs=4)]: Done  13 tasks      | elapsed:    9.6s
[Parallel(n_jobs=4)]: Done  14 tasks      | elapsed:    9.8s
[Parallel(n_jobs=4)]: Done  15 tasks      | elapsed:   11.0s
[Parallel([...]
{'fraction_non_alphanumeric': 0.1008601787822567, 'fraction_numerical': 0.020914150784280654, 'mean_word_length': 4.094501718213058, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '65703', 'license': 'MIT', 'n_tokens_mistral': 2177, 'n_tokens_neox': 2071, 'n_words': 525, 'path': 'note/00-data-qry/starai-preprocess.ipynb', 'repo_name': 'eliavw/missmercs', 'size': '5971'}