<commit_before>#!/usr/bin/env python2.7
# encoding: utf-8

"""
An implementation of rough clustering using rough set theory and the algorithm outlined by Voges, Pope & Brown, 2002,
"Cluster Analysis of Marketing Data Examining On-line Shopping Orientation: A Comparison of k-means and
Rough Clustering Approaches"

@options
max_clusters - integer corresponding to number of clusters to return
max_d - Maximum intra-entity distance to consider before stopping further clustering
if max_d is not specified, then algorithm determines max_d based on intra-entity distance statistics (25th percentile)

@note
The algorithm determines the optimal distance D for final clustering based on option 'objective' which maximizes :
		"lower" : sum of lower approximations (default) - maximum entity uniqueness across all clusters at distance D
		"coverage" : total # of entites covered by all clusters - maximum number of entities across all clusters at distance D
		"ratio" : ratio of lower/coverage - maximum ratio of unique entities to total entities across all clusters at distance D
		"all" : return clusters at every distance D from [0 - self.total_entities]

@author <NAME>
@copyright 2016
"""

#Externals
import time
import itertools
import operator
from collections import Counter
import numpy as npy
from copy import deepcopy

class roughCluster():

	def __init__(self,input_data,max_clusters,objective="lower",max_d=None):

		# Clustering output vars
		self.data = input_data
		self.distance = {}
		self.all_keys = {}
		self.clusters = []
		self.sum_upper = []
		self.sum_lower = []
		self.cluster_list = []
		self.total_entities = 0
		self.pruned = {}
		self.optimal = {}
		self.opt_d = None

		self.debug = False
		self.small = 1.0e-10

		# Clustering options
		self.minD = None					# Minimum intra-entity distance to perform clustering over
		self.maxD = max_d					# Maximum intra-entity distance to perform clustering over
		self.objective = objective			# Objective to maximize for optimal clustering distance D
		self.max_clusters = [max_clusters]	# Number of clusters to return

	def transformFeatures(self):

		"""
		Validation method for data types in feature space and transformation of float, string --> integer

		:arg self.data : dictionary of <feature_name> : feature vector list pairs of types (str, int, float)
		:return: self.dataT : dictionary containing integer transformation of all features in feature space
		"""
		# TODO finish this transformation method

		for keyname in self.data:

			if any(isinstance(n,float) for n in self.data[keyname][:]):
				locs = [0.0, float(npy.percentile(self.data[keyname],25)),float(npy.percentile(self.data[keyname],50)),
				float(npy.percentile(self.data[keyname],75)),float(npy.percentile(self.data[keyname],80)),
				float(npy.percentile(self.data[keyname],100))]
				for n in range(len(self.data[keyname])):
					for i,val in enumerate(locs[0:-1]):
						if (self.data["payload"][keyname][n]) >= val and (self.data["payload"][keyname][n] < locs[i+1]):
							self.data[keyname].append(i+1)

			elif any(isinstance(n,int) for n in self.data[keyname][:]):
				pass
				# Don't do anything to feature
			elif any(isinstance(n, (str, unicode)) for n in self.data[keyname][:]):
				pass
				# TODO map strings to ints
			else:
				raise Exception("Input features must be type int, float, or string")

		return

	def getEntityDistances(self):

		"""
		Compute intra-entity distance matrix for all unique entities in input

		uses self.data
		:return: self.distance : intra-entity distances for all unique (lower traingular) pairs of entities
		"""

		header = self.data.keys()
		data_length = len(self.data[header[0]])

		t1 = time.time()

		# Enumerate entire distance matrix
		for k in range(0,data_length):
			self.distance[str(k)] = {str(j) : sum([abs(self.data[val][k]-self.data[val][j]) for val in header])<commit_msg>added rough clustering class and test files<commit_after>#!/usr/bin/env python2.7
# encoding: utf-8

"""
An implementation of rough clustering using rough set theory and the algorithm outlined by Voges, Pope & Brown, 2002,
"Cluster Analysis of Marketing Data Examining On-line Shopping Orientation: A Comparison of k-means and
Rough Clustering Approaches"

This algorithm takes as input a feature set of type integer

@options
max_clusters - integer corresponding to number of clusters to return
max_d - Maximum intra-entity distance to consider before stopping further clustering
if max_d is not specified, then algorithm determines max_d based on intra-entity distance statistics (25th percentile)

@note
The algorithm determines the optimal distance D for final clustering based on option 'objective' which maximizes :
		"lower" : sum of lower approximations (default) - maximum entity uniqueness across all clusters at distance D
		"coverage" : total # of entites covered by all clusters - maximum number of entities across all clusters at distance D
		"ratio" : ratio of lower/coverage - maximum ratio of unique entities to total entities across all clusters at distance D
		"all" : return clusters at every distance D from [0 - self.total_entities]

@author <NAME>
@copyright 2016
"""

#Externals
import time
import itertools
import operator
from collections import Counter
import numpy as npy
from copy import deepcopy

class roughCluster():

	def __init__(self,input_data,max_clusters,objective="lower",max_d=None):

		# Clustering output vars
		self.data = input_data
		self.distance = {}
		self.all_keys = {}
		self.clusters = []
		self.sum_upper = []
		self.sum_lower = []
		self.cluster_list = []
		self.total_entities = 0
		self.pruned = {}
		self.optimal = {}
		self.opt_d = None

		self.debug = False
		self.small = 1.0e-10

		# Clustering options
		self.minD = None					# Minimum intra-entity distance to perform clustering over
		self.maxD = max_d					# Maximum intra-entity distance to perform clustering over
		self.objective = objective			# Objective to maximize for optimal clustering distance D
		self.max_clusters = [max_clusters]	# Number of clusters to return

	def getEntityDistances(self):

		"""
		Compute intra-entity distance matrix for all unique entities in input

		uses self.data
		:return: self.distance : intra-entity distances for all unique (lower traingular) pairs of entities
		"""

		header = self.data.keys()
		data_length = len(self.data[header[0]])

		t1 = time.time()

		# Enumerate entire distance matrix
		for k in range(0,data_length):
			self.distance[str(k)] = {str(j) : sum([abs(self.data[val][k]-self.data[val][j]) for val in header])
{'fraction_non_alphanumeric': 0.0921834526893189, 'fraction_numerical': 0.009294529940575957, 'mean_word_length': 4.314979757085021, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2388628', 'n_tokens_mistral': 2153, 'n_tokens_neox': 1892, 'n_words': 701}