<commit_before>from __future__ import division
import sys
import inspect
import json
import itertools
from collections import defaultdict
from datetime import datetime

import numpy as np
from sklearn import cluster, preprocessing, manifold, decomposition
from scipy.spatial import distance


class KeplerMapper(object):
    # With this class you can build topological networks from (high-dimensional) data.
    #
    # 1)   	Fit a projection/lens/function to a dataset and transform it.
    #     	For instance "mean_of_row(x) for x in X"
    # 2)   	Map this projection with overlapping intervals/hypercubes.
    #    		Cluster the points inside the interval
    #    		(Note: we cluster on the inverse image/original data to lessen projection loss).
    #    		If two clusters/nodes have the same members (due to the overlap), then:
    #    		connect these with an edge.
    # 3)  	Visualize the network using HTML and D3.js.
    #
    # functions
    # ---------
    # fit_transform:   Create a projection (lens) from a dataset
    # map:         	Apply Mapper algorithm on this projection and build a simplicial complex
    # visualize:    	Turns the complex dictionary into a HTML/D3.js visualization

    def __init__(self, verbose=2):
        self.verbose = verbose
        self.chunk_dist = []
        self.overlap_dist = []
        self.d = []
        self.nr_cubes = 0
        self.overlap_perc = 0
        self.clusterer = False

    def fit_transform(self, X, projection="sum", scaler=preprocessing.MinMaxScaler(), distance_matrix=False):<commit_msg>bug fix: had to run fit_transform first else no self.projection set yet<commit_after>from __future__ import division
import sys
import inspect
import json
import itertools
from collections import defaultdict
from datetime import datetime

import numpy as np
from sklearn import cluster, preprocessing, manifold, decomposition
from scipy.spatial import distance


class KeplerMapper(object):
    # With this class you can build topological networks from (high-dimensional) data.
    #
    # 1)   	Fit a projection/lens/function to a dataset and transform it.
    #     	For instance "mean_of_row(x) for x in X"
    # 2)   	Map this projection with overlapping intervals/hypercubes.
    #    		Cluster the points inside the interval
    #    		(Note: we cluster on the inverse image/original data to lessen projection loss).
    #    		If two clusters/nodes have the same members (due to the overlap), then:
    #    		connect these with an edge.
    # 3)  	Visualize the network using HTML and D3.js.
    #
    # functions
    # ---------
    # fit_transform:   Create a projection (lens) from a dataset
    # map:         	Apply Mapper algorithm on this projection and build a simplicial complex
    # visualize:    	Turns the complex dictionary into a HTML/D3.js visualization

    def __init__(self, verbose=2):
        self.verbose = verbose
        self.chunk_dist = []
        self.overlap_dist = []
        self.d = []
        self.nr_cubes = 0
        self.overlap_perc = 0
        self.clusterer = False
        self.projection = None

    def fit_transform(self, X, projection="sum", scaler=preprocessing.MinMaxScaler(), distance_matrix=False):
{'fraction_non_alphanumeric': 0.07025178737954615, 'fraction_numerical': 0.004973577867578489, 'mean_word_length': 2.8538922155688624, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '721783', 'n_tokens_mistral': 916, 'n_tokens_neox': 873, 'n_words': 342}