<jupyter_start><jupyter_text>mimlsvmmi = mulit-instanceml = multi-labelsvm = svmAs described in Shen et al 2009: http://ieeexplore.ieee.org/document/5346261/ good params: {'svm_degree': 3, 'svm_kernel': 'poly', 'medoid_normalization': None, 'nb_medoids_ratio': 0.2, 'svm_gamma': 'auto', 'vectorizer_norm': None, 'svm_c': 1}<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import gc
import nltk
import os
import re
import pickle
import sklearn
import sys
import string
import warnings

from nltk import TextTilingTokenizer

from sklearn.calibration import CalibratedClassifierCV
from sklearn.metrics import f1_score, precision_score, recall_score
from sklearn.model_selection import cross_val_score, GridSearchCV,ParameterGrid, train_test_split
from sklearn.multiclass import OneVsRestClassifier
from sklearn.preprocessing import MultiLabelBinarizer, StandardScaler,MinMaxScaler
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer,TfidfVectorizer
from sklearn.svm import SVC

from joblib import Parallel, delayed

from tqdm import *

%matplotlib inline
%load_ext autoreload
%autoreload 1
src_dir = os.path.join(os.getcwd(), os.pardir, '../../')
sys.path.append(src_dir)
%aimport src.data.movielens_20m_imdb
%aimport src.helpers.labels,src.helpers.neighbours, src.helpers.segments,src.helpers.embeddings
%aimport src.utils.dataframes, src.utils.clusters, src.utils.metrics, src.utils.distances, src.utils.plotting
from src.data.movielens_20m_imdb import load_df_or_get_from_cache
from src.helpers.labels import truncate_labels
from src.helpers.segments import make_distance_matrix_for_segments,vectorize_segments
from src.helpers.embeddings import read_glove_wiki_weighted

from src.utils.dataframes import sample_rows
from src.utils.metrics import ranking
from src.utils.clusters import k_medoids
from src.utils.distances import hausdorff
from src.utils.plotting import plot_micro_f1_at_k
MODELS_ROOT = os.path.abspath("../../../models/ranking/movielens-mimlsvm-embeddings/")
INTERIM_DATA_ROOT = os.path.abspath("../../../data/interim/movielens-ml20m-imdb/")
PATH_TO_PROCESSED_FILE = os.path.abspath('../../../data/processed/movielens-20m-imdb-tags-and-synopses-2017-12-20.csv')

# CONFIGS

SEED= 42
W=20 # Pseudosentence size (in words) - not specified in the paper, taken from TextTiling default values
K=10 # Size (in sentences) of the block used in the block comparison method - not specified in the paper, taken from TextTiling default values
MAX_NB_WORDS=500
EMBEDDINGS_DIM=100
SAMPLE_FRAC= 0.3
docs_df = load_df_or_get_from_cache(PATH_TO_PROCESSED_FILE,INTERIM_DATA_ROOT)
sentence_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
docs_df['sentences'] = docs_df['synopsis'].map(lambda row: sentence_tokenizer.tokenize(row))
%%time
docs_df['num_sentences'] = docs_df['sentences'].map( lambda sents: len(sents))
docs_df.head()
docs_df.iloc[0]['sentences'][:5]
docs_df.describe()
tok = TextTilingTokenizer(w=W, k=K)
def extract_segments(candidates):
    
    try:
        # we must manually insert "\n\n" because this is how 
        # texttilingtokenizer requires candidate boundaries to be 
        # represented.
        segments = tok.tokenize("\n\n".join(candidates))
    except ValueError:
        # this happens when the candidate list is too small for the 
        # text tiling tokenizer to be able to find segments. so just return
        # the original sentences.
        segments= candidates
        
    # now remove the artificially added chars
    segments = [segment.replace("\n\n"," ").strip() for segment in segments]
    
    return segments
if os.path.isfile(INTERIM_DATA_ROOT+"/docs_df_with_segments-{}.p".format(MAX_NB_WORDS)):
    print('cache hit')
    docs_df = pickle.load(open(INTERIM_DATA_ROOT+"/docs_df_with_segments-{}.p".format(MAX_NB_WORDS),"rb"))
else:
    docs_df['segments'] = docs_df['sentences'].map(lambda candidates: extract_segments(candidates))
    pickle.dump(docs_df,open(INTERIM_DATA_ROOT+"/docs_df_with_segments-{}.p".format(MAX_NB_WORDS),"wb"))<jupyter_output>cache hit<jupyter_text>sample for grid search<jupyter_code>sample_df = docs_df.sample(int(SAMPLE_FRAC*len(docs_df)))
segments = sample_df['segments'].values
documents = sample_df['synopsis'].values
labels = sample_df["tags"].map(lambda tagstring: tagstring.split(","))
labels = truncate_labels(labels,10)
# segments, documents and labelsets are defined outside of the parameterGrid loop
# because they're the same for every configuration    
segments_train, segments_val, documents_train, documents_val, Y_train, Y_val = train_test_split(segments,
                                                                                               documents,
                                                                                               labels,
                                                                                               test_size=0.15)

mlb = MultiLabelBinarizer()
mlb.fit(labels)

Y_train = mlb.transform(Y_train)
Y_val = mlb.transform(Y_val)

print('total number of train documents: {}'.format(len(documents_train)))
print('total number of validation documents: {}'.format(len(documents_val)))
print("total number of unique tags: {} ".format(len(mlb.classes_)))
vect = TfidfVectorizer(max_features=MAX_NB_WORDS)
vect.fit(documents)

feature_names = vect.get_feature_names()
idf = vect.idf_
idf_index = dict(zip(vect.get_feature_names(), idf))
# analyzer = preprocess + tokenize
tokenize_func = vect.build_analyzer()

def tokenize(string):
    return tokenize_func(string)
%%time
X_train_tok = list()

for document_segments in segments_train:
    tokenized_segments = [tokenize(segment) for segment in document_segments]
    X_train_tok.append(tokenized_segments)
%%time
X_val_tok = list()

for document_segments in segments_val:
    tokenized_segments = [tokenize(segment) for segment in document_segments]
    X_val_tok.append(tokenized_segments)<jupyter_output><empty_output><jupyter_text>transform into embeddings<jupyter_code>embeddings_index = read_glove_wiki_weighted(
    d=EMBEDDINGS_DIM,
    weight_index=idf_index)
def build_bag_of_weighted_embeddings(tokens): 
    out = [embeddings_index[token] for token in tokens if token in embeddings_index.keys()]
    return np.mean(np.array(out),axis=0)    
cache_path = INTERIM_DATA_ROOT+"/mimlsvm/mimlsvm-embeddings/X_train_boe_segments-sample-{}.p".format(SAMPLE_FRAC)

if os.path.isfile(cache_path):
    print('cache hit')
    X_train_boe_segments = pickle.load(open(cache_path,'rb'))
else:
    X_train_boe_segments = list()

    for tokenized_document_segments in X_train_tok:
        document_segments_boe = list()

        with warnings.catch_warnings():
            warnings.filterwarnings('error')
            for seg in tokenized_document_segments:
                if len(seg) == 0:
                    print('empty segment')
                    continue
                else:
                    try:
                        boe=build_bag_of_weighted_embeddings(seg)
                    except RuntimeWarning:
                        print('segment entirely with OOV words')
                        continue

                    document_segments_boe.append(boe)

        X_train_boe_segments.append(document_segments_boe)

    pickle.dump(X_train_boe_segments,open(cache_path,"wb"))
cache_path = INTERIM_DATA_ROOT+"/mimlsvm/mimlsvm-embeddings/X_val_boe_segments-sample-{}.p".format(SAMPLE_FRAC)

if os.path.isfile(cache_path):
    print('cache hit')
    X_val_boe_segments = pickle.load(open(cache_path,'rb'))
else:
    X_val_boe_segments = list()

    for tokenized_document_segments in X_val_tok:
        document_segments_boe = list()

        with warnings.catch_warnings():
            warnings.filterwarnings('error')
            for seg in tokenized_document_segments:
                if len(seg) == 0:
                    print('empty segment')
                    continue
                else:
                    try:
                        boe=build_bag_of_weighted_embeddings(seg)
                    except RuntimeWarning:
                        print('segment entirely with OOV words')
                        continue

                    document_segments_boe.append(boe)

        X_val_boe_segments.append(document_segments_boe)

    pickle.dump(X_val_boe_segments,open(cache_path,"wb"))
def make_distance_matrix_for_embedding_segments(vectorized_segments, distance='hausdorff'):
    """
    Returns the distance matrix for the documents having the given segments.

    :param vectorized_segments: array of size M, where each element is a "bag" of segments: matrix of
        shape (*,NUM_FEATURES), and each row on this matrix is the TF-IDF vector for one segment.
    :param distance: how to compare the two bags
    :return: pairwise distance matrix (MxM matrix)
    """
    if distance.lower().strip() != 'hausdorff':
        raise Exception("Only 'hausdorff' distance supported right now.")

    num_samples = len(vectorized_segments)

    distance_function = lambda a, b: hausdorff(a, b)

    distance_matrix = np.zeros((num_samples, num_samples))

    for i, segments_for_document_a in enumerate(vectorized_segments):
        for j, segments_for_document_b in enumerate(vectorized_segments):
            distance = distance_function(segments_for_document_a, segments_for_document_b)
            distance_matrix[i][j] = distance

    return distance_matrix
%%time

path_to_cache = INTERIM_DATA_ROOT+"/mimlsvm/mimlsvm-embeddings/distance-matrix-train-sample-{}.p".format(SAMPLE_FRAC)

if os.path.isfile(path_to_cache):
    print('cache hit')
    dist_matrix_train = pickle.load(open(path_to_cache,"rb"))
else:
    print('Fitting distance matrix')
    dist_matrix_train = make_distance_matrix_for_embedding_segments(X_train_boe_segments)
    pickle.dump(dist_matrix_train, open(path_to_cache, "wb"))
def make_train_dataset(distance_matrix, medoid_indices):
    """
    Returns a matrix where element Aij contains the distance from sample i to medoid j.

    :param distance_matrix: MxM matrix with pairwise distances
    :param medoid_indices: array of length N containing the indices of the medoids for each cluster
    :return: distances to medoids (MxN matrix)
    """

    return distance_matrix[:,medoid_indices]
def make_validation_dataset(source_vectorized_segments, medoid_vectorized_segments):
    """
    Calculates the distances from every source_document (reprsented by its segments) to every medoid
    document (also represented by its segments) using the hausdorff distance.
    
    Returns a matrix where element Aij contains the distance from sample i to medoid j.

    :param source_vectorized_segments: array of length M, where each element is a matrix with one row
        for every segment in a source document
    :param medoid_vectorized_segments: array of length N where each element is a matrix with one row
        for every segment in a medoid document
    :return: distances to medoids (MxN matrix)
    """
    
    num_val_samples = len(source_vectorized_segments)
    num_medoids = len(medoid_vectorized_segments)
    
    val_dataset = np.zeros((num_val_samples,num_medoids))    
    
    for i,source_segments in enumerate(source_vectorized_segments):
        for j,medoid_segments in enumerate(medoid_vectorized_segments):
            val_dataset[i][j] = hausdorff(source_segments,medoid_segments)
            
    return np.array(val_dataset)
# calculating medoids
NORMALIZATION = ['standard','minmax',None] # not specified in the paper
SAMPLE_TO_NB_MEDOIDS_RATIO = [0.2,0.3,0.4] # not specified in the paper, but taken from MIMLSVM canonical implementation
# classification
SVM_KERNEL=['rbf','linear','poly'] # not specified in the paper, but taken from MIMLSVM canonical implementation
SVM_GAMMA=['auto'] # not specified in the paper, but taken from MIMLSVM canonical implementation
SVM_C= [0.1,1,10]# not specified in the paper, but taken from MIMLSVM canonical implementation
SVM_DEGREE=[3,4,5]
grid_search_parameters = [
    {
        'medoid_normalization':  NORMALIZATION,
        'svm_kernel': ['poly'],
        'svm_c':SVM_C,
        'svm_degree' :SVM_DEGREE,
        'svm_gamma':SVM_GAMMA,
        'nb_medoids_ratio': SAMPLE_TO_NB_MEDOIDS_RATIO
    },
    {
        'medoid_normalization':  NORMALIZATION,
        'svm_kernel': ['rbf'],
        'svm_c':SVM_C,
        'svm_degree' :[None],
        'svm_gamma':SVM_GAMMA,
        'nb_medoids_ratio': SAMPLE_TO_NB_MEDOIDS_RATIO
    },
    {
        'medoid_normalization':  NORMALIZATION,
        'svm_kernel': ['linear'],
        'svm_c':SVM_C,
        'svm_gamma':SVM_GAMMA,
        'svm_degree' :[None],
        'nb_medoids_ratio': SAMPLE_TO_NB_MEDOIDS_RATIO
    }
]
print('total number of configurations to test: {}'.format(len(ParameterGrid(grid_search_parameters))))
import warnings
warnings.filterwarnings('ignore')

for (i,configuration) in tqdm(enumerate(ParameterGrid(grid_search_parameters))):
                 
    # nb_medoids depends upon the dataset length
    ratio = configuration['nb_medoids_ratio']
    nb_medoids = int(len(documents_train) * ratio)
    
    # these are the document segments to be used as medoids
    medoids_indices_train = k_medoids(dist_matrix_train,nb_medoids)[0]

    # a matrix where element Aij contains the distance from sample i to medoid j.
    X_train = make_train_dataset(dist_matrix_train,medoids_indices_train)
    
    # VALIDATION SET

    fitted_medoids = list()
    for medoid_index in medoids_indices_train:
        fitted_medoids.append(X_train_boe_segments[medoid_index]) 
    
    X_val = make_validation_dataset(X_val_boe_segments,fitted_medoids)     
        
    svm = SVC(kernel=configuration['svm_kernel'],
            gamma=configuration['svm_gamma'],
            C=configuration['svm_c'],
            degree=configuration['svm_degree'])
    
    clf = OneVsRestClassifier(CalibratedClassifierCV(svm,cv=2),n_jobs=-1)        

    if configuration['medoid_normalization'] == 'standard':      
        scaler = StandardScaler()
        X_train_final = scaler.fit_transform(X_train)
        X_val_final = scaler.transform(X_val)
    elif configuration['medoid_normalization'] == 'minmax':
        scaler = MinMaxScaler()
        X_train_final = scaler.fit_transform(X_train)
        X_val_final = scaler.transform(X_val)
    else:
        X_train_final = X_train
        X_val_final = X_val
    
    # y_train was defined outside the loop    
    clf.fit(X_train,Y_train)
    
    # train score
    Y_pred_train = clf.predict_proba(X_train)
    
    # validation score
    Y_pred_val = clf.predict_proba(X_val)  
    
    print("iter: {}, configuration: {}\n".format(i,configuration))
    
    ks = [1,2,3,4,5,6,7,8,9,10]

    for k in ks:
#         print("train micro-F1 @{}: {}".format(k,ranking.micro_f1_at_k(Y_train,Y_pred_train,k=k,normalize=True)))
        print("validation micro-F1 @{}: {}".format(k,ranking.micro_f1_at_k(Y_val,Y_pred_val,k=k,normalize=True)))<jupyter_output>0it [00:00, ?it/s][A
1it [02:14, 134.20s/it]<jupyter_text>train the final model<jupyter_code>segments = docs_df['segments'].values
documents = docs_df['synopsis'].values
labels = docs_df["tags"].map(lambda tagstring: tagstring.split(","))
# segments, documents and labelsets are defined outside of the parameterGrid loop
# because they're the same for every configuration    
segments_train, segments_val, documents_train, documents_val, Y_train, Y_val = train_test_split(segments,
                                                                                               documents,
                                                                                               labels,
                                                                                               test_size=0.15)

mlb = MultiLabelBinarizer()
mlb.fit(labels)

Y_train = mlb.transform(Y_train)
Y_val = mlb.transform(Y_val)

print('total number of train documents: {}'.format(len(documents_train)))
print('total number of validation documents: {}'.format(len(documents_val)))
print("total number of unique tags: {} ".format(len(mlb.classes_)))
vect = TfidfVectorizer(max_features=MAX_NB_WORDS)
vect.fit(documents)

feature_names = vect.get_feature_names()
idf = vect.idf_
idf_index = dict(zip(vect.get_feature_names(), idf))
# analyzer = preprocess + tokenize
tokenize_func = vect.build_analyzer()

def tokenize(string):
    return tokenize_func(string)
%%time
X_train_tok = list()

for document_segments in segments_train:
    tokenized_segments = [tokenize(segment) for segment in document_segments]
    X_train_tok.append(tokenized_segments)
%%time
X_val_tok = list()

for document_segments in segments_val:
    tokenized_segments = [tokenize(segment) for segment in document_segments]
    X_val_tok.append(tokenized_segments)<jupyter_output><empty_output><jupyter_text>no need to load embeddings index again<jupyter_code>if os.path.isfile(INTERIM_DATA_ROOT+"/mimlsvm/mimlsvm-embeddings/X_train_boe_segments.p"):
    print('cache hit')
    X_train_boe_segments = pickle.load(open(INTERIM_DATA_ROOT+"/mimlsvm/mimlsvm-embeddings/X_train_boe_segments.p",'rb'))
else:
    X_train_boe_segments = list()

    for tokenized_document_segments in X_train_tok:
        document_segments_boe = list()

        with warnings.catch_warnings():
            warnings.filterwarnings('error')
            for seg in tokenized_document_segments:
                if len(seg) == 0:
                    print('empty segment')
                    continue
                else:
                    try:
                        boe=build_bag_of_weighted_embeddings(seg)
                    except RuntimeWarning:
                        print('segment entirely with OOV words')
                        continue

                    document_segments_boe.append(boe)

        X_train_boe_segments.append(document_segments_boe)

    pickle.dump(X_train_boe_segments,open(INTERIM_DATA_ROOT+"/mimlsvm/mimlsvm-embeddings/X_train_boe_segments.p","wb"))
if os.path.isfile(INTERIM_DATA_ROOT+"/mimlsvm/mimlsvm-embeddings/X_val_boe_segments.p"):
    print('cache hit')
    X_val_boe_segments = pickle.load(open(INTERIM_DATA_ROOT+"/mimlsvm/mimlsvm-embeddings/X_val_boe_segments.p",'rb'))
else:
    X_val_boe_segments = list()

    for tokenized_document_segments in X_val_tok:
        document_segments_boe = list()

        with warnings.catch_warnings():
            warnings.filterwarnings('error')
            for seg in tokenized_document_segments:
                if len(seg) == 0:
                    print('empty segment')
                    continue
                else:
                    try:
                        boe=build_bag_of_weighted_embeddings(seg)
                    except RuntimeWarning:
                        print('segment entirely with OOV words')
                        continue

                    document_segments_boe.append(boe)

        X_val_boe_segments.append(document_segments_boe)

    pickle.dump(X_val_boe_segments,open(INTERIM_DATA_ROOT+"/mimlsvm/mimlsvm-embeddings/X_val_boe_segments.p","wb"))<jupyter_output>cache hit<jupyter_text>no need to redefine `make_distance_matrix_for_embedding_segments`, `make_train_dataset` and `make_val_dataset`<jupyter_code>%%time

path_to_cache = INTERIM_DATA_ROOT+"/mimlsvm/mimlsvm-embeddings/distance-matrix-train.p"

if os.path.isfile(path_to_cache):
    print('cache hit')
    dist_matrix_train = pickle.load(open(path_to_cache,"rb"))
else:
    print('Fitting distance matrix')

    dist_matrix_train = make_distance_matrix_for_embedding_segments(X_train_boe_segments)
    pickle.dump(dist_matrix_train, open(path_to_cache, "wb"))<jupyter_output>cache hit
CPU times: user 100 ms, sys: 272 ms, total: 372 ms
Wall time: 3.45 s<jupyter_text>set winning parameters and train on the whole dataset iter: 16, configuration: {'svm_degree': 4, 'svm_kernel': 'poly', 'svm_c': 10, 'medoid_normalization': 'standard', 'nb_medoids_ratio': 0.3, 'svm_gamma': 'auto'}<jupyter_code>final_parameters = [
    {
        'medoid_normalization':  ['standard'],
        'svm_kernel': ['poly'],
        'svm_c':[10],
        'svm_degree' :[4],
        'svm_gamma':['auto'],
        'nb_medoids_ratio': [0.3],
    }
]
for (i,configuration) in enumerate(ParameterGrid(final_parameters)):
                 
    # nb_medoids depends upon the dataset length
    ratio = configuration['nb_medoids_ratio']
    nb_medoids = int(len(documents_train) * ratio)
    
    # these are the document segments to be used as medoids
    medoids_indices_train = k_medoids(dist_matrix_train,nb_medoids)[0]

    # a matrix where element Aij contains the distance from sample i to medoid j.
    X_train = make_train_dataset(dist_matrix_train,medoids_indices_train)
    
    # VALIDATION SET

    fitted_medoids = list()
    for medoid_index in medoids_indices_train:
        fitted_medoids.append(X_train_boe_segments[medoid_index]) 
    
    X_val = make_validation_dataset(X_val_boe_segments,fitted_medoids)     
        
    svm = SVC(kernel=configuration['svm_kernel'],
            gamma=configuration['svm_gamma'],
            C=configuration['svm_c'],
            degree=configuration['svm_degree'])
    
    clf = OneVsRestClassifier(CalibratedClassifierCV(svm,cv=2),n_jobs=-1)        

    if configuration['medoid_normalization'] == 'standard':      
        scaler = StandardScaler()
        X_train_final = scaler.fit_transform(X_train)
        X_val_final = scaler.transform(X_val)
    elif configuration['medoid_normalization'] == 'minmax':
        scaler = MinMaxScaler()
        X_train_final = scaler.fit_transform(X_train)
        X_val_final = scaler.transform(X_val)
    else:
        X_train_final = X_train
        X_val_final = X_val
    
    # y_train was defined outside the loop    
    clf.fit(X_train,Y_train)
    
    # train score
    Y_pred_train = clf.predict_proba(X_train)
    
    # validation score
    Y_pred_val = clf.predict_proba(X_val)  
    
    print("iter: {}, configuration: {}\n".format(i,configuration))
    
    ks = [1,2,3,4,5,6,7,8,9,10]

    for k in ks:
#         print("train micro-F1 @{}: {}".format(k,ranking.micro_f1_at_k(Y_train,Y_pred_train,k=k,normalize=True)))
        print("validation micro-F1 @{}: {}".format(k,ranking.micro_f1_at_k(Y_test,Y_pred_val,k=k,normalize=True)))    
ks = [1,2,3,4,5,6,7,8,9,10]

for k in ks:
#         print("train micro-F1 @{}: {}".format(k,ranking.micro_f1_at_k(Y_train,Y_pred_train,k=k,normalize=True)))
    print("validation micro-F1 @{}: {}".format(k,ranking.micro_f1_at_k(Y_val,Y_pred_val,k=k,normalize=True)))<jupyter_output>validation micro-F1 @1: 0.17737556561085974
validation micro-F1 @2: 0.18476791347453808
validation micro-F1 @3: 0.18340348767288034
validation micro-F1 @4: 0.18190024825095916
validation micro-F1 @5: 0.17704561911658218
validation micro-F1 @6: 0.17075386012715713
validation micro-F1 @7: 0.1640838650865998
validation micro-F1 @8: 0.15989036089538602
validation micro-F1 @9: 0.1577396076837077
validation micro-F1 @10: 0.15500183217295713<jupyter_text>save the final model<jupyter_code>pickle.dump(clf, open(MODELS_ROOT.rstrip('/')+"/model.p", "wb"))
plt.clf()
img = plt.gcf()
ax = plt.gca()
validation_scores = [
validation micro-F1 @1: 0.17737556561085974
validation micro-F1 @2: 0.18476791347453808
validation micro-F1 @3: 0.18340348767288034
validation micro-F1 @4: 0.18190024825095916
validation micro-F1 @5: 0.17704561911658218
validation micro-F1 @6: 0.17075386012715713
validation micro-F1 @7: 0.1640838650865998
validation micro-F1 @8: 0.15989036089538602
validation micro-F1 @9: 0.1577396076837077
validation micro-F1 @10: 0.15500183217295713
]
plot_micro_f1_at_k(validation_scores,ax)
plt.gcf().set_size_inches(7,5)
plt.gca().legend_.remove()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09917147878483555, 'fraction_numerical': 0.023767679303707424, 'mean_word_length': 3.5418091980235653, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '253918', 'license': 'MIT', 'n_tokens_mistral': 8273, 'n_tokens_neox': 7781, 'n_words': 1519, 'path': 'social-tags/notebooks/movielens-imdb/ranking/mimlsvm-bag-of-embeddings.ipynb', 'repo_name': 'queirozfcom/auto-tagger', 'size': '23898'}