<commit_before>import math
from itertools import chain
import operator
import bisect
import itertools
from collections import defaultdict

from sklearn import preprocessing, tree, cross_validation
import numpy as np

from base import gob, utils
from base.utils import coord_in_miles
import explore.peek


def logify(x,fudge=1):
    return math.log(x+fudge,2)

def unlogify(x,fudge=1):
    return 2**x - fudge

def _scaled_local(x):
    return x if x is not None else .3

@gob.mapper()
def nebr_vect(user):
    for nebr in user['nebrs']:
        flags = [nebr['kind'] >>i & 1 for i in range(3)]
        logged = [logify(nebr[k]) for k in ('mdist','folc','frdc')]
        others = [
            _scaled_local(nebr['lofrd']),
            _scaled_local(nebr['lofol']),
            int(bool(nebr['prot'])),
            logify(coord_in_miles(user['mloc'],nebr),fudge=.01),
        ]
        yield flags + logged + others


def vects_as_mat(vects):
    # convert vects to a scaled numpy array
    vects_ = np.fromiter( chain.from_iterable(vects), np.float32 )
    vects_.shape = (len(vects_)//10),10
    X = np.array(vects_[:,:-1])
    y = np.array(vects_[:,-1])
    #scaler = preprocessing.Scaler().fit(X)

    # In final system, you will want to only fit the training data.
    #return scaler.transform(X), y
    return X,y


@gob.mapper(all_items=True)
def nebr_clf(vects):
    X, y = vects_as_mat(vects)
    clf = tree.DecisionTreeRegressor(max_depth=8)
    clf.fit(X,y)
    yield clf


class Predictor(object):
    def predict(self,nebrs_d):
        "return the index of a neighbor or (eventually) a coordinate"
        raise NotImplementedError


class Omni(Predictor):
    def predict(self,nebrs_d):
        values = [v[-1] for v in nebrs_d['vects']]
        return values.index(min(values))

class Nearest(Predictor):
    def predict(self,nebrs_d):
        return np.argmin(nebrs_d['pred_dists'])

class Last(Predictor):
    def predict(self,nebrs_d):
        return len(nebrs_d['strange_prob'])-1

class FacebookMLE(Predictor):
    def predict(self,nebrs_d):
        probs = np.sum(np.log(nebrs_d['contact_prob']),axis=0)
        return np.argmax(probs+nebrs_d['strange_prob'])


class NearestMLE(Predictor):
    def __init__(self,count):
        self.count = count

    def predict(self,nebrs_d):
        dists = sorted(
                    enumerate(nebrs_d['pred_dists']),
                    key=operator.itemgetter(1)
                    )
        best = [d[0] for d in dists[:self.count]]
        probs_ =    (
                    nebrs_d['contact_prob'][x][y]
                    for x in xrange(len(dists))
                    for y in xrange(len(dists))
                    if x in best and y in best
                    )
        probs = np.fromiter(probs_,float)
        probs.shape = len(best),len(best)

        sums = np.sum(np.log10(probs),axis=0)
        index, prob = max(enumerate(sums), key=operator.itemgetter(1))
        return best[index]

class FriendLoc(Predictor):
    def __init__(self,env,loc_factor,tz_factor=0,force_loc=False):
        self.env = env
        self.cutoffs,self.curves = zip(*tuple(self.env.load('vect_fit')))
        self.loc_factor = loc_factor
        self.tz_factor = tz_factor
        self.force_loc = force_loc

    def predict(self,nebrs_d):
        if self.force_loc and nebrs_d['gnp'] and nebrs_d['gnp']['mdist']<25:
            return len(nebrs_d['vects'])
        mat = nebrs_d['dist_mat']
        probs = np.empty_like(mat)
        for index,pred in enumerate(nebrs_d['pred_dists']):
            curve = self.curves[bisect.bisect(self.cutoffs,pred)-1]
            probs[index,:] = explore.peek.contact_curve(mat[index,:],*curve)
        total_probs = (
            np.sum(np.log(probs),axis=0) +
            nebrs_d['strange_prob'] +
            self.tz_factor*nebrs_d['tz_prob'] +
            self.loc_factor*nebrs_d['location_prob']
            )
        return np.argmax(total_probs)


class FriendLocWeights(Predictor):
    def __init__(self,env,weights):
        self.env = env
        self.cutoffs,self.curves = zip(*tuple(self.env.load('vect_fit')))
        self.weights = weights

    def predict(self,nebrs_d):

        mat = nebrs_d['dist_mat']
        probs = np.empty_like(mat)
        for index,pred in enumerate(nebrs_d['pred_dists']):
            curve = self.curves[bisect.bisect(self.cutoffs,pred)-1]
            probs[index,:] = explore.peek.contact_curve(mat[index,:],*curve)
        prob_sum = np.sum(np.log(probs),axis=0)
        w = self.weights
        total_probs = (
            w[0]*prob_sum +
            w[1]*prob_sum/len(nebrs_d['nebrs']) +
            w[2]*nebrs_d['strange_prob'] +
            w[3]*nebrs_d['tz_prob'] +
            w[4]*nebrs_d['location_prob']
            )
        return np.argmax(total_probs)


def _calc_dists(nebrs_d):
    gnp = nebrs_d['gnp']
    lats = [r['lat'] for r in nebrs_d['nebrs']]
    lngs = [r['lng'] for r in nebrs_d['nebrs']]
    all_lats = lats+[gnp['lat']] if gnp else lats
    all_lngs = lngs+[gnp['lng']] if gnp else lngs
    lat1,lat2 = np.meshgrid(all_lats,lats)
    lng1,lng2 = np.meshgrid(all_lngs,lngs)
    return utils.np_haversine(lng1,lng2,lat1,lat2)


class Predictors(object):
    def __init__(self,env):
        self.env = env
        '''
        self.classifiers = dict(
            omni=Omni(),
            nearest=Nearest(),
            last=Last(),
            backstrom=FacebookMLE(),
            friendloc_plain=FriendLoc(env,0),
            friendloc_field=FriendLoc(env,1),
            friendloc_tz=FriendLoc(env,1,1),
            friendloc_cut=FriendLoc(env,0,force_loc=True),
        )
        '''
        steps = [0,.5,1,2,25]
        self.classifiers = {
            vect:FriendLocWeights(env,vect)
            for vect in itertools.product(steps,repeat=5)
        }
        self.nebr_clf = next(env.load('nebr_clf','pkl'))

    def _mdist_curves(self):
        data = list(self.env.load('mdist_curves','mp'))
        return dict(
            curve_fn = [np.poly1d(d['coeffs']) for d in data],<commit_msg>comment out silly weight thing<commit_after>import math
from itertools import chain
import operator
import bisect
import itertools
from collections import defaultdict

from sklearn import preprocessing, tree, cross_validation
import numpy as np

from base import gob, utils
from base.utils import coord_in_miles
import explore.peek


def logify(x,fudge=1):
    return math.log(x+fudge,2)

def unlogify(x,fudge=1):
    return 2**x - fudge

def _scaled_local(x):
    return x if x is not None else .3

@gob.mapper()
def nebr_vect(user):
    for nebr in user['nebrs']:
        flags = [nebr['kind'] >>i & 1 for i in range(3)]
        logged = [logify(nebr[k]) for k in ('mdist','folc','frdc')]
        others = [
            _scaled_local(nebr['lofrd']),
            _scaled_local(nebr['lofol']),
            int(bool(nebr['prot'])),
            logify(coord_in_miles(user['mloc'],nebr),fudge=.01),
        ]
        yield flags + logged + others


def vects_as_mat(vects):
    # convert vects to a scaled numpy array
    vects_ = np.fromiter( chain.from_iterable(vects), np.float32 )
    vects_.shape = (len(vects_)//10),10
    X = np.array(vects_[:,:-1])
    y = np.array(vects_[:,-1])
    #scaler = preprocessing.Scaler().fit(X)

    # In final system, you will want to only fit the training data.
    #return scaler.transform(X), y
    return X,y


@gob.mapper(all_items=True)
def nebr_clf(vects):
    X, y = vects_as_mat(vects)
    clf = tree.DecisionTreeRegressor(max_depth=8)
    clf.fit(X,y)
    yield clf


class Predictor(object):
    def predict(self,nebrs_d):
        "return the index of a neighbor or (eventually) a coordinate"
        raise NotImplementedError


class Omni(Predictor):
    def predict(self,nebrs_d):
        values = [v[-1] for v in nebrs_d['vects']]
        return values.index(min(values))

class Nearest(Predictor):
    def predict(self,nebrs_d):
        return np.argmin(nebrs_d['pred_dists'])

class Last(Predictor):
    def predict(self,nebrs_d):
        return len(nebrs_d['strange_prob'])-1

class FacebookMLE(Predictor):
    def predict(self,nebrs_d):
        probs = np.sum(np.log(nebrs_d['contact_prob']),axis=0)
        return np.argmax(probs+nebrs_d['strange_prob'])


class NearestMLE(Predictor):
    def __init__(self,count):
        self.count = count

    def predict(self,nebrs_d):
        dists = sorted(
                    enumerate(nebrs_d['pred_dists']),
                    key=operator.itemgetter(1)
                    )
        best = [d[0] for d in dists[:self.count]]
        probs_ =    (
                    nebrs_d['contact_prob'][x][y]
                    for x in xrange(len(dists))
                    for y in xrange(len(dists))
                    if x in best and y in best
                    )
        probs = np.fromiter(probs_,float)
        probs.shape = len(best),len(best)

        sums = np.sum(np.log10(probs),axis=0)
        index, prob = max(enumerate(sums), key=operator.itemgetter(1))
        return best[index]

class FriendLoc(Predictor):
    def __init__(self,env,loc_factor,tz_factor=0,force_loc=False):
        self.env = env
        self.cutoffs,self.curves = zip(*tuple(self.env.load('vect_fit')))
        self.loc_factor = loc_factor
        self.tz_factor = tz_factor
        self.force_loc = force_loc

    def predict(self,nebrs_d):
        if self.force_loc and nebrs_d['gnp'] and nebrs_d['gnp']['mdist']<25:
            return len(nebrs_d['vects'])
        mat = nebrs_d['dist_mat']
        probs = np.empty_like(mat)
        for index,pred in enumerate(nebrs_d['pred_dists']):
            curve = self.curves[bisect.bisect(self.cutoffs,pred)-1]
            probs[index,:] = explore.peek.contact_curve(mat[index,:],*curve)
        total_probs = (
            np.sum(np.log(probs),axis=0) +
            nebrs_d['strange_prob'] +
            self.tz_factor*nebrs_d['tz_prob'] +
            self.loc_factor*nebrs_d['location_prob']
            )
        return np.argmax(total_probs)


class FriendLocWeights(Predictor):
    def __init__(self,env,weights):
        self.env = env
        self.cutoffs,self.curves = zip(*tuple(self.env.load('vect_fit')))
        self.weights = weights

    def predict(self,nebrs_d):

        mat = nebrs_d['dist_mat']
        probs = np.empty_like(mat)
        for index,pred in enumerate(nebrs_d['pred_dists']):
            curve = self.curves[bisect.bisect(self.cutoffs,pred)-1]
            probs[index,:] = explore.peek.contact_curve(mat[index,:],*curve)
        prob_sum = np.sum(np.log(probs),axis=0)
        w = self.weights
        total_probs = (
            w[0]*prob_sum +
            w[1]*prob_sum/len(nebrs_d['nebrs']) +
            w[2]*nebrs_d['strange_prob'] +
            w[3]*nebrs_d['tz_prob'] +
            w[4]*nebrs_d['location_prob']
            )
        return np.argmax(total_probs)


def _calc_dists(nebrs_d):
    gnp = nebrs_d['gnp']
    lats = [r['lat'] for r in nebrs_d['nebrs']]
    lngs = [r['lng'] for r in nebrs_d['nebrs']]
    all_lats = lats+[gnp['lat']] if gnp else lats
    all_lngs = lngs+[gnp['lng']] if gnp else lngs
    lat1,lat2 = np.meshgrid(all_lats,lats)
    lng1,lng2 = np.meshgrid(all_lngs,lngs)
    return utils.np_haversine(lng1,lng2,lat1,lat2)


class Predictors(object):
    def __init__(self,env):
        self.env = env
        self.classifiers = dict(
            omni=Omni(),
            nearest=Nearest(),
            last=Last(),
            backstrom=FacebookMLE(),
            friendloc_plain=FriendLoc(env,0),
            friendloc_field=FriendLoc(env,1),
            friendloc_tz=FriendLoc(env,1,1),
            friendloc_cut=FriendLoc(env,0,force_loc=True),
        )
        '''
        steps = [0,.5,1,2,25]
        self.classifiers = {
            vect:FriendLocWeights(env,vect)
            for vect in itertools.product(steps,repeat=5)
        }
        '''
        self.nebr_clf = next(env.load('nebr_clf','pkl'))

    def _mdist_curves(self):
        data = list(self.env.load('mdist_curves','mp'))
        return dict(
            curve_fn = [np.poly1d(d['coeffs']) for d in data],
{'fraction_non_alphanumeric': 0.13417951042611062, 'fraction_numerical': 0.009890381603890218, 'mean_word_length': 2.546916106401637, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '860806', 'n_tokens_mistral': 4511, 'n_tokens_neox': 4426, 'n_words': 769}