<commit_before># importing modules/ libraries
import pandas as pd
import random
import numpy as np

# create a sample of prior orders
orders_df = pd.read_csv("Data/orders.csv")
s = round(3214874 * 0.1)
i = sorted(random.sample(list(orders_df[orders_df["eval_set"]=="prior"].index), s))
orders_df.loc[i,:].to_csv("Data/orders_prior_sample.csv", index = False)

# create a sample of train orders
s = round(131209 * 0.1)
j = sorted(random.sample(list(orders_df[orders_df["eval_set"]=="train"].index), s))
orders_df.loc[j,:].to_csv("Data/orders_train_sample.csv", index = False)

# create a sample of test orders
s = round(75000 * 0.1)
k = sorted(random.sample(list(orders_df[orders_df["eval_set"]=="test"].index), s))
orders_df.loc[k,:].to_csv("Data/orders_test_sample.csv", index = False)

# create a sample of prior order products
order_products_prior_df = pd.read_csv('Data/order_products__prior.csv', index_col = 'order_id')
order_products_prior_df.loc[orders_df.loc[i,:]['order_id'],:].to_csv("Data/order_products_prior_sample.csv", index = False)

# create a sample of train order products
order_products_train_df = pd.read_csv('Data/order_products__train.csv', index_col = 'order_id')
order_products_train_df.loc[orders_df.loc[j,:]['order_id'],:].to_csv("Data/order_products_train_sample.csv", index = False)
<commit_msg>fix: Remove code to create sample for test orders<commit_after># importing modules/ libraries
import pandas as pd
import random
import numpy as np

# create a sample of prior orders
orders_df = pd.read_csv("Data/orders.csv")
s = round(3214874 * 0.1)
i = sorted(random.sample(list(orders_df[orders_df["eval_set"]=="prior"].index), s))
orders_df.loc[i,:].to_csv("Data/orders_prior_sample.csv", index = False)

# create a sample of train orders
s = round(131209 * 0.1)
j = sorted(random.sample(list(orders_df[orders_df["eval_set"]=="train"].index), s))
orders_df.loc[j,:].to_csv("Data/orders_train_sample.csv", index = False)

# create a sample of prior order products
order_products_prior_df = pd.read_csv('Data/order_products__prior.csv', index_col = 'order_id')
order_products_prior_df.loc[orders_df.loc[i,:]['order_id'],:].to_csv("Data/order_products_prior_sample.csv", index = False)

# create a sample of train order products
order_products_train_df = pd.read_csv('Data/order_products__train.csv', index_col = 'order_id')
order_products_train_df.loc[orders_df.loc[j,:]['order_id'],:].to_csv("Data/order_products_train_sample.csv", index = False)

{'fraction_non_alphanumeric': 0.14995957962813258, 'fraction_numerical': 0.016572352465642683, 'mean_word_length': 7.277591973244147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '181232', 'n_tokens_mistral': 909, 'n_tokens_neox': 870, 'n_words': 171}