<jupyter_start><jupyter_text>FoNS Datathon 2021<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import *

train_descriptors = pd.read_csv("train_descriptors.csv")
train_mord3d = pd.read_csv("train_mord3d.csv")
train_morgan = pd.read_csv("train_morgan.csv")
train_rdk = pd.read_csv("train_rdk.csv")

train_crystals = pd.read_csv("train_crystals.csv")
train_distances = pd.read_csv("train_distances.csv")
train_centroid_distances = pd.read_csv("train_centroid_distances.csv")
test_descriptors = pd.read_csv("test_descriptors.csv")
test_mord3d = pd.read_csv("test_mord3d.csv")
test_morgan = pd.read_csv("test_morgan.csv")
test_rdk = pd.read_csv("test_rdk.csv")<jupyter_output><empty_output><jupyter_text>Data pre-processing<jupyter_code>train_descriptors_full = train_descriptors.iloc[:, 3:-2].dropna(axis= 1, how="any")
train_descriptors_full.shape
test_descriptors_full = test_descriptors[train_descriptors_full.columns]
test_descriptors_full.shape
train_PCA = decomposition.PCA(n_components=.95)
scaler_for_PCA = preprocessing.StandardScaler()
train_descriptors_PCA = train_PCA.fit_transform(scaler_for_PCA.fit_transform(train_descriptors_full))
test_descriptors_PCA = train_PCA.transform(scaler_for_PCA.transform(test_descriptors_full))
print(train_descriptors_PCA.shape, test_descriptors_PCA.shape)<jupyter_output>(13449, 124) (3363, 124)<jupyter_text>A (very!!) simple and bad model<jupyter_code>target = "a"
model = linear_model.LinearRegression()
model.fit(train_descriptors_PCA, train_crystals[target])
predictions = model.predict(train_descriptors_PCA)
plt.plot(train_crystals[target], predictions, "bo")
plt.xlabel("True values")
plt.ylabel("Predicted values")
plt.title("Predicted z-value against true values")
plt.show()<jupyter_output><empty_output><jupyter_text>Cross-validation to evaluate the model's performance<jupyter_code>cv_scores = model_selection.cross_val_score(
    model, 
    train_descriptors_PCA, 
    train_crystals[target], 
    scoring="neg_mean_absolute_error")

print("Average negative MAE:", round(np.mean(cv_scores), 3))
print("Standard deviation:", round(np.std(cv_scores), 3))<jupyter_output>Average negative MAE: -3.258
Standard deviation: 0.05<jupyter_text>Outputting predictions for automated checking Link: https://github.com/stevenkbennett/fons_datathon_testing<jupyter_code>predictions = model.predict(test_descriptors_PCA)
np.savetxt("task_x_predictions.csv", predictions)
# Alternatively, if using pandas, you can use the following:
pd.DataFrame(predictions).to_csv("task_x_predictions.csv", header=False, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11435062476334722, 'fraction_numerical': 0.014009844755774327, 'mean_word_length': 9.827868852459016, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '663353', 'license': 'MIT', 'n_tokens_mistral': 993, 'n_tokens_neox': 957, 'n_words': 132, 'path': 'Getting_started.ipynb', 'repo_name': 'Imperial-CMTH/digifab-datathon', 'size': '2641'}