<jupyter_start><jupyter_text>Imports and global variables<jupyter_code>import pandas as pd
import os
from sklearn.model_selection import train_test_split
import numpy as np
DATA_FOLDER = "Data"<jupyter_output><empty_output><jupyter_text>Data cleanup and transformation<jupyter_code>df = pd.read_csv(os.path.join(DATA_FOLDER, "corefx-issues-train.tsv"), sep="\t")
df.head()<jupyter_output><empty_output><jupyter_text>Remove nan values<jupyter_code>df = df.replace(np.nan, '', regex=True)
df[df.isnull().any(axis=1)]<jupyter_output><empty_output><jupyter_text>Transform label values to numbers<jupyter_code>df.Area.unique()
# Create a dictionary to easily go between label and its corresponding numeral value
lookup = {}
for i, area in enumerate(df.Area.unique()):
    lookup[area] = i
    lookup[i] = area
lookup
# transform the Area column to be numeric using the lookup dict created above
df.Area = df.Area.apply(lambda x: lookup[x])
df.Area
labels = df.Area
titles = df.Title
descriptions = df.Description<jupyter_output><empty_output><jupyter_text>Transform data<jupyter_code># Create a text column that will be the concatenation of the title and description columns
text = []
for title, description in zip(titles, descriptions):
    text.append(" ".join(title.split()) + " " + " ".join(description.split()))
len(labels)
len(text)
text[:5]
labels[:5]
[lookup[x] for x in labels[:5]]<jupyter_output><empty_output><jupyter_text>Create new DataFrame with transformed data<jupyter_code>df = pd.DataFrame(list(zip(text, labels)), columns =['Text', 'Label'])
df.head()
# Split into traning and validation sets, use stratify to have same distribution of data in both sets
train_df, val_df = train_test_split(df, test_size=0.2, random_state=42, stratify=labels)
print(len(train_df))
print(len(val_df))
# Save the CSVs
train_df.to_csv(os.path.join(DATA_FOLDER, "corefx_cleaned_train.csv"), index=False)
val_df.to_csv(os.path.join(DATA_FOLDER, "corefx_cleaned_val.csv"), index=False)
# Save the lookup dict
import json

with open(os.path.join(DATA_FOLDER, 'lookup.json'), 'w') as f:
    json.dump(lookup, f, indent=4)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12757009345794393, 'fraction_numerical': 0.004205607476635514, 'mean_word_length': 7.079245283018868, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '161105', 'license': 'MIT', 'n_tokens_mistral': 751, 'n_tokens_neox': 741, 'n_words': 178, 'path': 'data_preprocessing.ipynb', 'repo_name': 'Ghisso/RobertaGithubIssuesClassification', 'size': '2140'}