<jupyter_start><jupyter_text>Prepare Item Matrix<jupyter_code>import pandas as pd
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
from ast import literal_eval
rawfile = "../input/all_output.json"
raw = pd.read_json(rawfile)
raw["id"] = raw["url"].apply(lambda url: url.split("/")[-1])
raw["joined"] = raw["joined"].apply(lambda joined: joined.split("Joined ")[1])
raw = raw.drop(columns=["url"])
raw.head()
import re
from unidecode import unidecode
from sklearn.preprocessing import MultiLabelBinarizer


def clean_column_name(name):
    old = unidecode(name.casefold())
    clean = re.sub("[^0-9a-zA-Z]+", '_', old)
    out = "in_" + clean
    return out


mlb = MultiLabelBinarizer()
mlb.fit(raw["categories"])
one_hot = mlb.transform(raw["categories"])
READABLE_LABELS = list(mlb.classes_)
COLUMN_LABELS = [clean_column_name(label) for label in READABLE_LABELS]
mat = pd.DataFrame(one_hot, columns=COLUMN_LABELS)
mat.head()
df = raw.join(mat)
ATTRIBUTES = ["id", "name", "price", "reactions", "stars", "joined", "categories"]
columns = ATTRIBUTES + COLUMN_LABELS
df = df[columns]
df = df.drop_duplicates(subset=["id"], keep="first")
df.head()
records = df.to_dict(orient="records")
for record in records:
    # record["categories"] = literal_eval(record["categories"])
    for col in COLUMN_LABELS:
        if record[col] == 0:
            del record[col]
print("Converted {} rows to sparse records.".format(len(records)))
import pickle


filename = "../input/talent.pkl"
pickle.dump((records, COLUMN_LABELS, READABLE_LABELS, ATTRIBUTES), open(filename, "wb"))
print("Wrote {} records and metadata to: {}.".format(len(df), filename))
# Example of reading the data
item_records, cols, labels, attrs = pickle.load(open(filename, "rb"))
new_df = pd.DataFrame(item_records)[attrs + cols].fillna(value=0)
new_df.head(3)
new_df[attrs].dtypes
print("Values in the categories series are: {}".format(type(new_df["categories"][0])))
sns.heatmap(pd.DataFrame(mat.copy().values, columns=READABLE_LABELS).corr())
plt.gcf().set_size_inches((12, 8))
plt.title("Category Correlations")
plt.show()
df[ATTRIBUTES].sort_values(by=["reactions"], ascending=False).head(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16427289048473967, 'fraction_numerical': 0.0058348294434470375, 'mean_word_length': 6.237012987012987, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '42157', 'license': 'MIT', 'n_tokens_mistral': 794, 'n_tokens_neox': 789, 'n_words': 169, 'path': 'recommenders/notebooks/01 Prepare Item Matrix.ipynb', 'repo_name': 'acmillinoistech/redcarpet', 'size': '2228'}