# Path: scorpion/sigmod/overlap.py
# class OverlapPenalty(object):
#   def __init__(self, domain, cdists, ddists, granularity=100):
#     """
#     Args:
#       domain: Orange.Domain object
#     """
#     self.domain = domain
#     self.cdists = cdists
#     self.ddists = ddists
#     self.granularity = granularity
#     self.funcs = {}
#     self.allcounts = {}
#     self.disc_nvals = {}
#     self.setup()
# 
#   def setup(self):
# 
#     funcs = {}
#     allcounts = {}
#     for attr in self.domain:
#       if attr.var_type == OType.Discrete:
#         func, counts = self.setup_discrete_attribute(attr)
#       else:
#         func, counts = self.setup_continuous_attribute(attr)
#       funcs[attr.name] = func
#       allcounts[attr.name] = counts
# 
#     self.funcs, self.allcounts = funcs, allcounts
# 
#   def setup_continuous_attribute(self, attr):
#     distribution = self.cdists[attr.name]
#     minv, maxv = distribution.min, distribution.max
#     if minv == maxv:
#       func = lambda v: 0
#       counts = np.zeros(1)
#     else:
#       def make_func(minv, block, gran):
#         def f(v):
#           return int(min(gran, max(0, math.ceil((v-minv)/block))))
#         return f
#       block = (maxv - minv) / float(self.granularity)
#       counts = np.zeros(self.granularity+1)
#       func = make_func(minv, block, self.granularity)
# 
#     return func, counts
# 
#   def setup_discrete_attribute(self, attr):
#     vals = self.ddists[attr.name].keys()
#     d = {val: idx for idx, val in enumerate(vals)}
#     
#     # values will be passed in as indices into vals
#     def func(v):
#       if v < 0 or v > len(vals) or v is None:
#         return len(vals)
#       return v
#     counts = np.zeros(len(vals)+1)
#     self.disc_nvals[attr.name] = len(vals)
#     return func, counts
# 
#   def reset_counts(self):
#     for counts in self.allcounts.values():
#       counts[:] = 0
# 
#   def continuous_idxs(self, attr, minv, maxv):
#     if isinstance(attr, basestring):
#       name = attr
#     else:
#       name = attr.name
# 
#     dist = self.cdists[name]
#     if minv <= dist.min and maxv >= dist.max:
#       return []
#     func = self.funcs[name]
#     return np.arange(func(minv),func(maxv)+1)
#   
#   def discrete_idxs(self, attr, vals):
#     if isinstance(attr, basestring):
#       name = attr
#     else:
#       name = attr.name
# 
#     if len(vals) == self.disc_nvals[name]:
#       return np.array([])
#     func = self.funcs[name]
#     return np.array(map(func, vals))
# 
#   def __call__(self, clusters, min_weight=0.7):
#     """
#     return weights to multiply to each cluster's influence
#     """
#     penalties = self.penalties(clusters)
#     weights = 1. - penalties
#     weights[weights <= min_weight] = min_weight
#     return weights
# 
#   def penalties(self, clusters):
#     """
#     Compute a penalty for each cluster
#     Return is normalzed to [0, 1]
#     """
#     self.reset_counts()
#     penalties = np.array(map(self.penalty, clusters))
#     if penalties.max() == 0:
#       return penalties
#     penalties /= penalties.max()
#     return penalties
# 
#   def penalty(self, cluster):
#     totals = {}
#     for col, (minv, maxv) in zip(cluster.cols, zip(*cluster.bbox)):
#       idxs = self.continuous_idxs(col, minv, maxv)
#       if len(idxs):
#         totals[col] = self.allcounts[col][idxs]
#         self.allcounts[col][idxs] += .5
# 
#     for col, vals in cluster.discretes.iteritems():
#       idxs = self.discrete_idxs(col, vals)
#       if len(idxs):
#         totals[col] = self.allcounts[col][idxs]
#         self.allcounts[col][idxs] += 1
# 
#     smooth = lambda counts: max(0, (counts - 0.5).max())
#     return sum(map(smooth, totals.values()))


import json
import math
import pdb
import random
import numpy as np
import sys
import time
from itertools import izip
from collections import *
from scipy.optimize import fsolve
from scorpion.util import *
from scorpion.sigmod.overlap import OverlapPenalty
  Ends up being faster than the above root-finding-based approaches
  """
  buckets_cache = {}

  def __init__(self, c_range, min_granularity=0, K=3, nblocks=100, learner=None):
    """
    Args
      min_granularity: minimum distance between adjacent intersections
      K: select top-k at each bucket
    """
    Frontier.__init__(self, c_range, min_granularity=min_granularity)


    self.seen = set()
    self.seen_clusters = set()

    self.nblocks = nblocks
    self.buckets = CheapFrontier.compute_normalized_buckets(nblocks, self.seen_clusters)
    self.buckets = self.buckets * r_vol(c_range) + c_range[0]
    self.nblocks = len(self.buckets)
    self.bests = defaultdict(list)   # bucket -> clusters
    self.K = K

    self.frontier = []
    self.frontier_hashes = []
    self.frontier_infs = []
    self.threshold = np.zeros(self.nblocks).astype(float)

    self.overlap = None
    if learner:

# Path: scorpion/sigmod/overlap.py
# class OverlapPenalty(object):
#   def __init__(self, domain, cdists, ddists, granularity=100):
#     """
#     Args:
#       domain: Orange.Domain object
#     """
#     self.domain = domain
#     self.cdists = cdists
#     self.ddists = ddists
#     self.granularity = granularity
#     self.funcs = {}
#     self.allcounts = {}
#     self.disc_nvals = {}
#     self.setup()
# 
#   def setup(self):
# 
#     funcs = {}
#     allcounts = {}
#     for attr in self.domain:
#       if attr.var_type == OType.Discrete:
#         func, counts = self.setup_discrete_attribute(attr)
#       else:
#         func, counts = self.setup_continuous_attribute(attr)
#       funcs[attr.name] = func
#       allcounts[attr.name] = counts
# 
#     self.funcs, self.allcounts = funcs, allcounts
# 
#   def setup_continuous_attribute(self, attr):
#     distribution = self.cdists[attr.name]
#     minv, maxv = distribution.min, distribution.max
#     if minv == maxv:
#       func = lambda v: 0
#       counts = np.zeros(1)
#     else:
#       def make_func(minv, block, gran):
#         def f(v):
#           return int(min(gran, max(0, math.ceil((v-minv)/block))))
#         return f
#       block = (maxv - minv) / float(self.granularity)
#       counts = np.zeros(self.granularity+1)
#       func = make_func(minv, block, self.granularity)
# 
#     return func, counts
# 
#   def setup_discrete_attribute(self, attr):
#     vals = self.ddists[attr.name].keys()
#     d = {val: idx for idx, val in enumerate(vals)}
#     
#     # values will be passed in as indices into vals
#     def func(v):
#       if v < 0 or v > len(vals) or v is None:
#         return len(vals)
#       return v
#     counts = np.zeros(len(vals)+1)
#     self.disc_nvals[attr.name] = len(vals)
#     return func, counts
# 
#   def reset_counts(self):
#     for counts in self.allcounts.values():
#       counts[:] = 0
# 
#   def continuous_idxs(self, attr, minv, maxv):
#     if isinstance(attr, basestring):
#       name = attr
#     else:
#       name = attr.name
# 
#     dist = self.cdists[name]
#     if minv <= dist.min and maxv >= dist.max:
#       return []
#     func = self.funcs[name]
#     return np.arange(func(minv),func(maxv)+1)
#   
#   def discrete_idxs(self, attr, vals):
#     if isinstance(attr, basestring):
#       name = attr
#     else:
#       name = attr.name
# 
#     if len(vals) == self.disc_nvals[name]:
#       return np.array([])
#     func = self.funcs[name]
#     return np.array(map(func, vals))
# 
#   def __call__(self, clusters, min_weight=0.7):
#     """
#     return weights to multiply to each cluster's influence
#     """
#     penalties = self.penalties(clusters)
#     weights = 1. - penalties
#     weights[weights <= min_weight] = min_weight
#     return weights
# 
#   def penalties(self, clusters):
#     """
#     Compute a penalty for each cluster
#     Return is normalzed to [0, 1]
#     """
#     self.reset_counts()
#     penalties = np.array(map(self.penalty, clusters))
#     if penalties.max() == 0:
#       return penalties
#     penalties /= penalties.max()
#     return penalties
# 
#   def penalty(self, cluster):
#     totals = {}
#     for col, (minv, maxv) in zip(cluster.cols, zip(*cluster.bbox)):
#       idxs = self.continuous_idxs(col, minv, maxv)
#       if len(idxs):
#         totals[col] = self.allcounts[col][idxs]
#         self.allcounts[col][idxs] += .5
# 
#     for col, vals in cluster.discretes.iteritems():
#       idxs = self.discrete_idxs(col, vals)
#       if len(idxs):
#         totals[col] = self.allcounts[col][idxs]
#         self.allcounts[col][idxs] += 1
# 
#     smooth = lambda counts: max(0, (counts - 0.5).max())
#     return sum(map(smooth, totals.values()))


# Path: scorpion/sigmod/frontier.py
import json
import math
import pdb
import random
import numpy as np
import sys
import time
from itertools import izip
from collections import *
from scipy.optimize import fsolve
from scorpion.util import *
from scorpion.sigmod.overlap import OverlapPenalty

  Ends up being faster than the above root-finding-based approaches
  """
  buckets_cache = {}

  def __init__(self, c_range, min_granularity=0, K=3, nblocks=100, learner=None):
    """
    Args
      min_granularity: minimum distance between adjacent intersections
      K: select top-k at each bucket
    """
    Frontier.__init__(self, c_range, min_granularity=min_granularity)


    self.seen = set()
    self.seen_clusters = set()

    self.nblocks = nblocks
    self.buckets = CheapFrontier.compute_normalized_buckets(nblocks, self.seen_clusters)
    self.buckets = self.buckets * r_vol(c_range) + c_range[0]
    self.nblocks = len(self.buckets)
    self.bests = defaultdict(list)   # bucket -> clusters
    self.K = K

    self.frontier = []
    self.frontier_hashes = []
    self.frontier_infs = []
    self.threshold = np.zeros(self.nblocks).astype(float)

    self.overlap = None
    if learner:

      self.overlap = OverlapPenalty(