<jupyter_start><jupyter_text>Target data:<jupyter_code>import pandas
target_df = pandas.read_csv('../data/model_data_2.csv')
target_df.head()
len(target_df)<jupyter_output><empty_output><jupyter_text>250374 different clients (unique clients). There a multiple policies (300K). Let's use the above dataset to split the pictures across the folder structure:<jupyter_code>training_one_df = target_df[(target_df['DESIRED_CL_CCPCEC']  == 1) & (target_df['sample'] == 'training')]
training_one_df.head()
len(training_one_df)
training_zero_df = target_df[(target_df['DESIRED_CL_CCPC']  == 0) & (target_df['sample'] == 'training')]
training_zero_df.head()
len(training_zero_df)
12556 + 187952
validation_one_df = target_df[(target_df['DESIRED_CL_CCPC']  == 1) & (target_df['sample'] == 'holdout')]
validation_zero_df = target_df[(target_df['DESIRED_CL_CCPC']  == 0) & (target_df['sample'] == 'holdout')]
training_one_df = training_one_df['FILE_NAME']
training_zero_df = training_zero_df['FILE_NAME']
validation_one_df = validation_one_df['FILE_NAME']
validation_zero_df = validation_zero_df['FILE_NAME']
training_one = training_one_df.tolist()
training_zero = training_zero_df.tolist()
validation_one = validation_one_df.tolist()
validation_zero = validation_zero_df.tolist()
'file_1886.png' in training_one
import os
from os import path
import shutil

src = "../imgs/"
dst = "../data/train/positive/"


files = [i for i in training_one]
for f in files:
    try:
        shutil.copy(src + str(f),dst)
    except:
        pass
import os
from os import path
import shutil

src = "../imgs/"
dst = "../data/train/negative/"


files = [i for i in training_zero]
for f in files:
    try:
        shutil.copy(src + str(f),dst)
    except:
        pass
import os
from os import path
import shutil

src = "../imgs/"
dst = "../data/validation/positive/"


files = [i for i in validation_one]
for f in files:
    try:
        shutil.copy(src + str(f),dst)
    except:
        pass
import os
from os import path
import shutil

src = "../imgs/"
dst = "../data/validation/negative/"


files = [i for i in validation_zero]
for f in files:
    try:
        shutil.copy(src + str(f),dst)
    except:
        pass
import os

path, dirs, files = next(os.walk("../data/train/positive/"))
file_count = len(files)
print(file_count)
print(len(training_one_df))
path, dirs, files = next(os.walk("../data/train/negative/"))
file_count = len(files)
print(file_count)
print(len(training_zero_df))
path, dirs, files = next(os.walk("../data/validation/positive/"))
file_count = len(files)
print(file_count)
print(len(validation_one_df))
path, dirs, files = next(os.walk("../data/validation/negative/"))
file_count = len(files)
print(file_count)
print(len(validation_zero_df))
8735 + 16139 + 2065 + 10482
12556 + 187952 + 3123 + 46743
import os

path, dirs, files = next(os.walk("../imgs/"))
file_count = len(files)
print(file_count)<jupyter_output>23010
{'fraction_non_alphanumeric': 0.1440329218106996, 'fraction_numerical': 0.024691358024691357, 'mean_word_length': 4.869215291750503, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '488425', 'license': 'MIT', 'n_tokens_mistral': 1104, 'n_tokens_neox': 1035, 'n_words': 232, 'path': 'code/Target-Preparation-v01.ipynb', 'repo_name': 'markeyser/Hackaton', 'size': '2916'}