<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon Apr 20 00:00:41 2015

@author: Caasar
"""
# -*- coding: utf-8 -*-
"""
Use a Bayes Inference algorithm to build a model supporting a simple solution
for the missing data problem.

I assume conditional independence of the provided columns given the clicks.
This allows for a fast inference with low memory requirements.

@author: <NAME>
"""

import numpy as np
from numpy.random import randn
from network import BayesNetwork, BayesAdaptStemmingNode, BayesCategoricalNode

path = 'data_recruiting_bi_data_challenge.csv'
TEST_SIZE = 50000
cols = np.arange(1,11)
dtypes = [("hotel_id",np.uint64),("city_id",np.uint64),("clicks",float),
         ("stars",float),("distance_to_center",float),("avg_price_hotel",float),
         ("rating",float),("nmbr_partners_index",float),
         ("avg_rel_saving",float),("avg_rank",float)]

data = np.genfromtxt(path,delimiter=',',dtype=dtypes,usecols=cols,names=True)

missing = np.isnan(data['clicks'])

unkown = data[missing]
known  = data[~missing]

# split for own validation
rndset = randn(len(known)).argsort()
test = known[rndset[:TEST_SIZE]]
train = known[rndset[TEST_SIZE:]]

def measure(true,pred):
    """
    Computes the provided weighted MSE performance measure.
    
    Parameters
    ----------
    true : numpy.ndarray
        The true clicks.
    pred : numpy.ndarray
        The predicted clicks.
    """
    w = np.log(true+1)+1
    w /= w.sum()
    diff = true-pred
    return np.inner(w,diff*diff)

netcost = lambda test, net : np.sqrt(measure(test['clicks'],net.mean_prediction(test)))

# %% Set network parameters and train models

nodes = [(BayesAdaptStemmingNode,{'label':'avg_rank','stems':64,'min_raise':5}),
         (BayesAdaptStemmingNode,{'label':'avg_rel_saving','stems':32}),
         (BayesAdaptStemmingNode,{'label':'nmbr_partners_index','stems':32}),
         (BayesAdaptStemmingNode,{'label':'avg_price_hotel','stems':32}), 
         (BayesAdaptStemmingNode,{'label':'distance_to_center','stems':16}), 
         (BayesAdaptStemmingNode,{'label':'rating','stems':32,'min_raise':1}),
         (BayesCategoricalNode,{'label':'city_id'}),
         (BayesCategoricalNode,{'label':'stars','ids':[0,1,2,3,4,5]}),
         (BayesAdaptStemmingNode,{'label':'clicks','stems':64,'min_raise':5})]

parents = {'rating':['clicks'], 'avg_rank':['clicks'],'stars':['clicks'],
           'avg_rel_saving':['clicks'],'nmbr_partners_index':['clicks'],
           'avg_price_hotel':['clicks'],'distance_to_center':['clicks'],
           'distance_to_center':['clicks'],'city_id':['clicks']}


ghost_sample = 1.0
weighting = lambda d: np.log(d['clicks']+1)+1
network = BayesNetwork(nodes,parents).learn(train,ghost_sample,weighting)

w = np.log(test['clicks']+1)+1
pred_baseline = np.inner(w,test['clicks'])/w.sum()
print('Baseline rmse: %.3f' % np.sqrt(measure(test['clicks'],pred_baseline)))
print('All models: %.3f' % netcost(test,network))<commit_msg>Added simple grid search for optimal parameters to main<commit_after># -*- coding: utf-8 -*-
"""
Created on Mon Apr 20 00:00:41 2015

@author: Caasar
"""
# -*- coding: utf-8 -*-
"""
Use a Bayes Inference algorithm to build a model supporting a simple solution
for the missing data problem.

I assume conditional independence of the provided columns given the clicks.
This allows for a fast inference with low memory requirements.

@author: <NAME>
"""

import numpy as np
from numpy.random import randn
from network import BayesNetwork, BayesAdaptStemmingNode, BayesCategoricalNode
from itertools import product

path = 'data_recruiting_bi_data_challenge.csv'
TEST_SIZE = 50000
cols = np.arange(1,11)
dtypes = [("hotel_id",np.uint64),("city_id",np.uint64),("clicks",float),
         ("stars",float),("distance_to_center",float),("avg_price_hotel",float),
         ("rating",float),("nmbr_partners_index",float),
         ("avg_rel_saving",float),("avg_rank",float)]

data = np.genfromtxt(path,delimiter=',',dtype=dtypes,usecols=cols,names=True)

missing = np.isnan(data['clicks'])

unkown = data[missing]
known  = data[~missing]

# split for own validation
rndset = randn(len(known)).argsort()
test = known[rndset[:TEST_SIZE]]
train = known[rndset[TEST_SIZE:]]

def measure(true,pred):
    """
    Computes the provided weighted MSE performance measure.
    
    Parameters
    ----------
    true : numpy.ndarray
        The true clicks.
    pred : numpy.ndarray
        The predicted clicks.
    """
    w = np.log(true+1)+1
    w /= w.sum()
    diff = true-pred
    return np.inner(w,diff*diff)

netcost = lambda test, net : np.sqrt(measure(test['clicks'],net.mean_prediction(test)))

# %% Set network parameters and train models

nodes = [(BayesAdaptStemmingNode,{'label':'avg_rank','stems':64,'min_raise':5}),
         (BayesAdaptStemmingNode,{'label':'avg_rel_saving','stems':32}),
         (BayesAdaptStemmingNode,{'label':'nmbr_partners_index','stems':32}),
         (BayesAdaptStemmingNode,{'label':'avg_price_hotel','stems':32}), 
         (BayesAdaptStemmingNode,{'label':'distance_to_center','stems':16}), 
         (BayesAdaptStemmingNode,{'label':'rating','stems':32,'min_raise':1}),
         (BayesCategoricalNode,{'label':'city_id'}),
         (BayesCategoricalNode,{'label':'stars','ids':[0,1,2,3,4,5]}),
         (BayesAdaptStemmingNode,{'label':'clicks','stems':64,'min_raise':5})]

parents = {'rating':['clicks'], 'avg_rank':['clicks'],'stars':['clicks'],
           'avg_rel_saving':['clicks'],'nmbr_partners_index':['clicks'],
           'avg_price_hotel':['clicks'],'distance_to_center':['clicks'],
           'distance_to_center':['clicks'],'city_id':['clicks']}


ghost_sample = 1.0
weighting = lambda d: np.log(d['clicks']+1)+1
network = BayesNetwork(nodes,parents).learn(train,ghost_sample,weighting)

w = np.log(test['clicks']+1)+1
pred_baseline = np.inner(w,test['clicks'])/w.sum()
full_cost = netcost(test,network)
print('Baseline rmse: %.3f' % np.sqrt(measure(test['clicks'],pred_baseline)))
print('All models: %.3f' % full_cost)


fields = {'avg_rank','avg_rel_saving','nmbr_partners_index','avg_price_hotel',
          'distance_to_center','rating','clicks'}

min_raise = [0,.5,1,5,10,20]
stems = [8,16,32,48,64,96,128]
costs = np.zeros((len(nodes),len(min_raise)*len(stems)))
for run in range(3):
    for i in range(len(nodes)):
        tnodes = list(nodes)
        mcl, odict = tnodes[i]
        if odict['label'] not in fields:
            continue
        for j,(r,s) in enumerate(product(min_raise,stems)):
            ndict = dict(odict)
            ndict['min_raise'] = r
            ndict['stems'] = s
            tnodes[i] = mcl, ndict
            print('%s: Setting min_raise=%d, stems=%d'%(ndict['label'],r,s))
            tnet = BayesNetwork(tnodes,parents).learn(train)
            tcost = netcost(test,tnet)
            print('\tcost %.3f (%.3f)'% (tcost,full_cost))
            if tcost < full_cost:
                nodes[i] = mcl, ndict
                full_cost = tcost
        
{'fraction_non_alphanumeric': 0.17562929061784896, 'fraction_numerical': 0.020451945080091533, 'mean_word_length': 4.480407523510972, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '541782', 'n_tokens_mistral': 2531, 'n_tokens_neox': 2408, 'n_words': 420}